thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Added functionaliy for eStorage buffers to work correctly

Browse source 
Signed-off-by: Alejandro Saucedo <axsauze@gmail.com>
This commit is contained in:
Alejandro Saucedo 2022-12-03 17:24:00 +00:00
parent 201e43b25d
commit f7a77ed521
No known key found for this signature in database
GPG key ID: 99EBB7E98B99695F
2 changed files with 19 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/OpTensorCopy.cpp
View file

@ -61,10 +61,20 @@ OpTensorCopy::postEval(const vk::CommandBuffer& /*commandBuffer*/)
{
KP_LOG_DEBUG("Kompute OpTensorCopy postEval called");
// Do not copy on CPU side if source is storage tensor
if (this->mTensors[0]->tensorType() == kp::Tensor::TensorTypes::eStorage)
{
KP_LOG_DEBUG("Kompute OpTensorCopy not copying tensor source given it's of eStorage type");
return;
}
void* data = this->mTensors[0]->rawData();
// Copy the data from the first tensor into all the tensors
for (size_t i = 1; i < this->mTensors.size(); i++) {
if (this->mTensors[i]->tensorType() == kp::Tensor::TensorTypes::eStorage) {
KP_LOG_DEBUG("Kompute OpTensorCopy not copying to tensor dest given it's of eStorage type");
continue;
}
this->mTensors[i]->setRawData(data);
}
}

14
src/Tensor.cpp
View file

@ -87,9 +87,11 @@ Tensor::rebuild(void* data,
}
this->allocateMemoryCreateGPUResources();
this->mapRawData();
memcpy(this->mRawData, data, this->memorySize());
if (this->tensorType() != Tensor::TensorTypes::eStorage) {
this->mapRawData();
memcpy(this->mRawData, data, this->memorySize());
}
}
Tensor::TensorTypes
@ -155,7 +157,7 @@ Tensor::mapRawData()
hostVisibleMemory = this->mStagingMemory;
} else {
KP_LOG_WARN(
"Kompute Tensor mapping data not supported on storage tensor");
"Kompute Tensor mapping data not supported on {} tensor", toString(this->tensorType()));
return;
}
@ -182,7 +184,7 @@ Tensor::unmapRawData()
hostVisibleMemory = this->mStagingMemory;
} else {
KP_LOG_WARN(
"Kompute Tensor mapping data not supported on storage tensor");
"Kompute Tensor mapping data not supported on {} tensor", toString(this->tensorType()));
return;
}
@ -520,7 +522,9 @@ Tensor::destroy()
}
// Unmap the current memory data
this->unmapRawData();
if (this->tensorType() != Tensor::TensorTypes::eStorage) {
this->unmapRawData();
}
if (this->mFreePrimaryBuffer) {
if (!this->mPrimaryBuffer) {