thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Added optensorSyncDevice and optensorcopy tests

Browse source
This commit is contained in:
Alejandro Saucedo 2020-09-06 12:52:45 +01:00
parent ec89fc6d56
commit 1b4e4b6b18
12 changed files with 398 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

50
src/OpTensorSyncDevice.cpp
View file

@ -1,4 +1,6 @@
#include "kompute/Tensor.hpp"
#include "kompute/operations/OpTensorSyncDevice.hpp"
namespace kp {
@ -21,8 +23,6 @@ OpTensorSyncDevice::OpTensorSyncDevice(
OpTensorSyncDevice::~OpTensorSyncDevice()
{
SPDLOG_DEBUG("Kompute OpTensorSyncDevice destructor started");
SPDLOG_DEBUG("Kompute OpTensorSyncDevice destroying staging tensors");
}
void
@ -30,17 +30,37 @@ OpTensorSyncDevice::init()
{
SPDLOG_DEBUG("Kompute OpTensorSyncDevice init called");
if (this->mTensors.size() < 2) {
if (this->mTensors.size() < 1) {
throw std::runtime_error(
"Kompute OpTensorSyncDevice called with less than 2 tensor");
"Kompute OpTensorSyncDevice called with less than 1 tensor");
}
for (std::shared_ptr<Tensor> tensor: this->mTensors) {
if (!tensor->isInit()) {
throw std::runtime_error("Kompute OpTensorSyncDevice tensor parameter has not been initialized");
if (tensor->isInit()) {
throw std::runtime_error("Kompute OpTensorSyncDevice: Tensor has already been initialized");
}
if (tensor->tensorType() == Tensor::TensorTypes::eStorage) {
throw std::runtime_error("Kompute OpTensorSyncDevice tensor parameter is of type storage and hence cannot be used to receive or pass data.");
throw std::runtime_error("Kompute OpTensorSyncLocal tensor parameter is of type TensorTypes::eStorage and hence cannot be used to receive or pass data.");
}
if (tensor->tensorType() == Tensor::TensorTypes::eDevice) {
std::shared_ptr<Tensor> stagingTensor = std::make_shared<Tensor>(
tensor->data(), Tensor::TensorTypes::eStaging);
stagingTensor->init(
this->mPhysicalDevice, this->mDevice);
stagingTensor->mapDataIntoHostMemory();
this->mStagingTensors.push_back(stagingTensor);
} else {
tensor->mapDataIntoHostMemory();
// We push a nullptr when no staging tensor is needed to match
// index number in array to have one to one mapping with tensors
this->mStagingTensors.push_back(nullptr);
}
}
}
@ -50,9 +70,10 @@ OpTensorSyncDevice::record()
{
SPDLOG_DEBUG("Kompute OpTensorSyncDevice record called");
// We iterate from the second tensor onwards and record a copy to all
for (size_t i = 1; i < this->mTensors.size(); i++) {
this->mTensors[i]->recordCopyFrom(this->mCommandBuffer, this->mTensors[0], false);
for (size_t i = 0; i < this->mTensors.size(); i++) {
if (this->mTensors[i]->tensorType() == Tensor::TensorTypes::eDevice) {
this->mTensors[i]->recordCopyFrom(this->mCommandBuffer, this->mStagingTensors[i], false);
}
}
}
@ -61,11 +82,10 @@ OpTensorSyncDevice::postSubmit()
{
SPDLOG_DEBUG("Kompute OpTensorSyncDevice postSubmit called");
// Copy the data from the first tensor into all the tensors
for (size_t i = 1; i < this->mTensors.size(); i++) {
this->mTensors[i]->setData(this->mTensors[0]->data());
}
// Remove all staging tensors as they are not required after operation
SPDLOG_DEBUG("Kompute OpTensorSyncDevice destroying staging tensors");
// TODO: This would cause issues if there is no CPU barrier
this->mStagingTensors.clear();
}
}