thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Moved destroy and rebuild functions to cpp

Browse source
This commit is contained in:
Alejandro Saucedo 2021-02-10 08:44:25 +00:00
parent 605a9d1d4a
commit c83ba44549
2 changed files with 132 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

124
src/Manager.cpp
View file

@ -350,5 +350,129 @@ Manager::tensor(
return tensor;
}
void
Manager::rebuild(std::vector<std::shared_ptr<kp::Tensor>> tensors,
bool syncDataToGPU)
{
SPDLOG_DEBUG("Kompute Manager rebuild triggered");
for (std::shared_ptr<Tensor> tensor : tensors) {
// False syncData to run all tensors at once instead one by one
this->rebuild(tensor, false);
}
if (syncDataToGPU) {
this->evalOpDefault<OpTensorSyncDevice>(tensors);
}
}
void
Manager::rebuild(std::shared_ptr<kp::Tensor> tensor,
bool syncDataToGPU)
{
SPDLOG_DEBUG("Kompute Manager rebuild Tensor triggered");
if (tensor->isInit()) {
tensor->freeMemoryDestroyGPUResources();
}
tensor->init(this->mPhysicalDevice, this->mDevice);
std::set<std::shared_ptr<Tensor>>::iterator it =
this->mManagedTensors.find(tensor);
if (it == this->mManagedTensors.end()) {
this->mManagedTensors.insert(tensor);
}
if (syncDataToGPU) {
this->evalOpDefault<OpTensorSyncDevice>({ tensor });
}
}
void
Manager::destroy(std::shared_ptr<kp::Tensor> tensor)
{
SPDLOG_DEBUG("Kompute Manager rebuild Tensor triggered");
if (tensor->isInit()) {
tensor->freeMemoryDestroyGPUResources();
}
// TODO: Confirm not limiting destroying tensors owned by this manager allowed
std::set<std::shared_ptr<Tensor>>::iterator it =
this->mManagedTensors.find(tensor);
if (it != this->mManagedTensors.end()) {
this->mManagedTensors.erase(tensor);
}
}
void
Manager::destroy(std::vector<std::shared_ptr<kp::Tensor>> tensors)
{
SPDLOG_DEBUG("Kompute Manager rebuild Tensor triggered");
for (std::shared_ptr<Tensor> tensor : tensors) {
this->destroy(tensor);
}
}
void
Manager::destroy(std::vector<std::shared_ptr<kp::Sequence>> sequences)
{
SPDLOG_DEBUG("Kompute Manager rebuild Sequence triggered");
for (std::shared_ptr<kp::Sequence> sequence : sequences) {
this->destroy(sequence);
}
}
void
Manager::destroy(std::shared_ptr<kp::Sequence> sequence)
{
SPDLOG_DEBUG("Kompute Manager rebuild Sequence triggered");
// Inefficient but required to delete by value
// Depending on the amount of named sequences created may be worth creating
// a set to ensure efficient delete.
for (std::unordered_map<std::string, std::shared_ptr<Sequence>>::iterator it = this->mManagedSequences.begin(); it != this->mManagedSequences.end(); it++) {
if (it->second == sequence) {
this->mManagedSequences.erase(it);
break;
}
}
if (sequence->isInit()) {
sequence->freeMemoryDestroyGPUResources();
}
}
void
Manager::destroy(const std::string& sequenceName)
{
SPDLOG_DEBUG("Kompute Manager rebuild Sequence triggered");
std::unordered_map<std::string, std::shared_ptr<Sequence>>::iterator
found = this->mManagedSequences.find(sequenceName);
if (found != this->mManagedSequences.end()) {
// We don't call destroy(sequence) as erasing sequence by name more efficient
if (found->second->isInit()) {
found->second->freeMemoryDestroyGPUResources();
}
this->mManagedSequences.erase(sequenceName);
}
}
void
Manager::destroy(const std::vector<std::string>& sequenceNames)
{
SPDLOG_DEBUG("Kompute Manager rebuild Sequence triggered");
for (const std::string& sequenceName : sequenceNames) {
this->destroy(sequenceName);
}
}
}

114
src/include/kompute/Manager.hpp
View file

@ -238,19 +238,7 @@ class Manager
* @param syncDataToGPU Whether to sync the data to GPU memory
*/
void rebuild(std::vector<std::shared_ptr<kp::Tensor>> tensors,
bool syncDataToGPU = true)
{
SPDLOG_DEBUG("Kompute Manager rebuild triggered");
for (std::shared_ptr<Tensor> tensor : tensors) {
// False syncData to run all tensors at once instead one by one
this->rebuild(tensor, false);
}
if (syncDataToGPU) {
this->evalOpDefault<OpTensorSyncDevice>(tensors);
}
}
bool syncDataToGPU = true);
/**
* Function that simplifies the common workflow of tensor initialisation. It
@ -262,26 +250,7 @@ class Manager
* @param syncDataToGPU Whether to sync the data to GPU memory
*/
void rebuild(std::shared_ptr<kp::Tensor> tensor,
bool syncDataToGPU = true)
{
SPDLOG_DEBUG("Kompute Manager rebuild Tensor triggered");
if (tensor->isInit()) {
tensor->freeMemoryDestroyGPUResources();
}
tensor->init(this->mPhysicalDevice, this->mDevice);
std::set<std::shared_ptr<Tensor>>::iterator it =
this->mManagedTensors.find(tensor);
if (it == this->mManagedTensors.end()) {
this->mManagedTensors.insert(tensor);
}
if (syncDataToGPU) {
this->evalOpDefault<OpTensorSyncDevice>({ tensor });
}
}
bool syncDataToGPU = true);
/**
* Destroy owned Vulkan GPU resources and free GPU memory for
@ -289,22 +258,7 @@ class Manager
*
* @param tensors Single tensor to rebuild
*/
void destroy(std::shared_ptr<kp::Tensor> tensor)
{
SPDLOG_DEBUG("Kompute Manager rebuild Tensor triggered");
if (tensor->isInit()) {
tensor->freeMemoryDestroyGPUResources();
}
// TODO: Confirm not limiting destroying tensors owned by this manager allowed
std::set<std::shared_ptr<Tensor>>::iterator it =
this->mManagedTensors.find(tensor);
if (it != this->mManagedTensors.end()) {
this->mManagedTensors.erase(tensor);
}
}
void destroy(std::shared_ptr<kp::Tensor> tensor);
/**
* Destroy owned Vulkan GPU resources and free GPU memory for
@ -312,14 +266,7 @@ class Manager
*
* @param tensors Single tensor to rebuild
*/
void destroy(std::vector<std::shared_ptr<kp::Tensor>> tensors)
{
SPDLOG_DEBUG("Kompute Manager rebuild Tensor triggered");
for (std::shared_ptr<Tensor> tensor : tensors) {
this->destroy(tensor);
}
}
void destroy(std::vector<std::shared_ptr<kp::Tensor>> tensors);
/**
* Destroy owned Vulkan GPU resources and free GPU memory for
@ -328,14 +275,7 @@ class Manager
*
* @param sequences Vector for shared ptrs with sequences to destroy
*/
void destroy(std::vector<std::shared_ptr<kp::Sequence>> sequences)
{
SPDLOG_DEBUG("Kompute Manager rebuild Sequence triggered");
for (std::shared_ptr<kp::Sequence> sequence : sequences) {
this->destroy(sequence);
}
}
void destroy(std::vector<std::shared_ptr<kp::Sequence>> sequences);
/**
* Destroy owned Vulkan GPU resources and free GPU memory for
@ -344,24 +284,7 @@ class Manager
*
* @param sequences Single sequence to rebuild
*/
void destroy(std::shared_ptr<kp::Sequence> sequence)
{
SPDLOG_DEBUG("Kompute Manager rebuild Sequence triggered");
// Inefficient but required to delete by value
// Depending on the amount of named sequences created may be worth creating
// a set to ensure efficient delete.
for (std::unordered_map<std::string, std::shared_ptr<Sequence>>::iterator it = this->mManagedSequences.begin(); it != this->mManagedSequences.end(); it++) {
if (it->second == sequence) {
this->mManagedSequences.erase(it);
break;
}
}
if (sequence->isInit()) {
sequence->freeMemoryDestroyGPUResources();
}
}
void destroy(std::shared_ptr<kp::Sequence> sequence);
/**
* Destroy owned Vulkan GPU resources and free GPU memory for
@ -369,21 +292,7 @@ class Manager
*
* @param sequenceName Single name of named sequence to destroy
*/
void destroy(const std::string& sequenceName)
{
SPDLOG_DEBUG("Kompute Manager rebuild Sequence triggered");
std::unordered_map<std::string, std::shared_ptr<Sequence>>::iterator
found = this->mManagedSequences.find(sequenceName);
if (found != this->mManagedSequences.end()) {
// We don't call destroy(sequence) as erasing sequence by name more efficient
if (found->second->isInit()) {
found->second->freeMemoryDestroyGPUResources();
}
this->mManagedSequences.erase(sequenceName);
}
}
void destroy(const std::string& sequenceName);
/**
* Destroy owned Vulkan GPU resources and free GPU memory for
@ -391,14 +300,7 @@ class Manager
*
* @param sequenceName Vector of sequence names to destroy
*/
void destroy(const std::vector<std::string>& sequenceNames)
{
SPDLOG_DEBUG("Kompute Manager rebuild Sequence triggered");
for (const std::string& sequenceName : sequenceNames) {
this->destroy(sequenceName);
}
}
void destroy(const std::vector<std::string>& sequenceNames);
private:
// -------------- OPTIONALLY OWNED RESOURCES