thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Added a buildTensor command to manager to simplify tensor creation workflows

Browse source
This commit is contained in:
Alejandro Saucedo 2020-09-10 08:35:27 +01:00
parent dcd8340b27
commit 4171786b6f
5 changed files with 147 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

12
README.md
View file

@ -60,8 +60,9 @@ int main() {
// You can allow Kompute to create the Vulkan components, or pass your existing ones
kp::Manager mgr; // Selects device 0 unless explicitly requested
auto tensorA = std::make_shared<kp::Tensor>(kp::Tensor({ 0, 1, 2 }));
auto tensorRhs = std::make_shared<kp::Tensor>(kp::Tensor({ 2, 4, 6 }));
// Creates tensor an initializes GPU memory (below we show more granularity)
auto tensorA = mgr.buildTensor({ 3, 4, 5 });
auto tensorB = mgr.buildTensor({ 0, 0, 0 });
// Define your shader as a string (using string literals for simplicity)
// (You can also pass the raw compiled bytes, or even path to file)
@ -80,12 +81,9 @@ int main() {
}
)");
// Create tensor data in GPU
mgr.evalOpDefault<kp::OpTensorCreate>({ tensorA, tensorB });
// Run Kompute operation on the parameters provided with dispatch layout
mgr.evalOpDefault<kp::OpMult<3, 1, 1>>(
{ tensorLhs, tensorRhs, tensorOut },
{ tensorA, tensorB },
true, // Whether to retrieve the output from GPU memory
std::vector<char>(shader.begin(), shader.end()));
@ -126,7 +124,7 @@ int main() {
auto tensorRhs = std::make_shared<kp::Tensor>(kp::Tensor({ 2., 4., 6. }));
auto tensorOut = std::make_shared<kp::Tensor>(kp::Tensor({ 0., 0., 0. }));
// Create tensors data in GPU
// Create tensors data explicitly in GPU with an operation
mgr.evalOpDefault<kp::OpTensorCreate>({ tensorLhs, tensorRhs, tensorOut });
// Run Kompute operation on the parameters provided with dispatch layout

161
single_include/kompute/Kompute.hpp
View file

@ -651,6 +651,74 @@ class Sequence
} // End namespace kp
namespace kp {
/**
Operation that creates tensor and manages the memory of the components
created
*/
class OpTensorCreate : public OpBase
{
public:
OpTensorCreate();
/**
* Default constructor with parameters that provides the bare minimum
* requirements for the operations to be able to create and manage their
* sub-components.
*
* @param physicalDevice Vulkan physical device used to find device queues
* @param device Vulkan logical device for passing to Algorithm
* @param commandBuffer Vulkan Command Buffer to record commands into
* @param tensors Tensors that will be used to create in operation.
* @param freeTensors Whether operation manages the memory of the Tensors
*/
OpTensorCreate(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer,
std::vector<std::shared_ptr<Tensor>> tensors);
/**
* Default destructor which in this case expects the parent class to free
* the tensors
*/
~OpTensorCreate() override;
/**
* In charge of initialising the primary Tensor as well as the staging
* tensor as required. It will only initialise a staging tensor if the
* Primary tensor is of type Device. For staging tensors it performs a
* mapDataIntoHostMemory which would perform immediately as opposed to
* on sequence eval/submission.
*/
void init() override;
/**
* Record runs the core actions to create the tensors. For device tensors
* it records a copyCommand to move the data from the staging tensor to the
* device tensor. The mapping for staging tensors happens in the init function
* not in the record function.
*/
void record() override;
/**
* Does not perform any preEval commands.
*/
virtual void preEval() override;
/**
* Performs a copy back into the main tensor to ensure that the data
* contained is the one that is now being stored in the GPU.
*/
virtual void postEval() override;
private:
// Never owned resources
std::vector<std::shared_ptr<Tensor>> mStagingTensors;
};
} // End namespace kp
#define KP_DEFAULT_SESSION "DEFAULT"
namespace kp {
@ -660,7 +728,6 @@ namespace kp {
*/
class Manager
{
private:
public:
/**
Base constructor and default used which creates the base resources
@ -757,6 +824,30 @@ class Manager
this->evalOp<T>(tensors, KP_DEFAULT_SESSION, std::forward<TArgs>(params)...);
}
/**
* Function that simplifies the common workflow of tensor creation and
* initialization. It will take the constructor parameters for a Tensor
* and will will us it to create a new Tensor and then create it using
* the OpCreateTensor command.
*
* @param data The data to initialize the tensor with
* @param tensorType The type of tensor to initialize
* @returns Initialized Tensor with memory Syncd to GPU device
*/
std::shared_ptr<Tensor> buildTensor(
const std::vector<float>& data,
Tensor::TensorTypes tensorType = Tensor::TensorTypes::eDevice)
{
SPDLOG_DEBUG("Kompute Manager createInitTensor triggered");
SPDLOG_DEBUG("Kompute Manager creating new tensor shared ptr");
std::shared_ptr<Tensor> tensor = std::make_shared<Tensor>(kp::Tensor(data, tensorType));
this->evalOpDefault<OpTensorCreate>({tensor});
return tensor;
}
private:
// -------------- OPTIONALLY OWNED RESOURCES
std::shared_ptr<vk::Instance> mInstance = nullptr;
@ -1531,74 +1622,6 @@ class OpMult : public OpAlgoBase<tX, tY, tZ>
namespace kp {
/**
Operation that creates tensor and manages the memory of the components
created
*/
class OpTensorCreate : public OpBase
{
public:
OpTensorCreate();
/**
* Default constructor with parameters that provides the bare minimum
* requirements for the operations to be able to create and manage their
* sub-components.
*
* @param physicalDevice Vulkan physical device used to find device queues
* @param device Vulkan logical device for passing to Algorithm
* @param commandBuffer Vulkan Command Buffer to record commands into
* @param tensors Tensors that will be used to create in operation.
* @param freeTensors Whether operation manages the memory of the Tensors
*/
OpTensorCreate(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer,
std::vector<std::shared_ptr<Tensor>> tensors);
/**
* Default destructor which in this case expects the parent class to free
* the tensors
*/
~OpTensorCreate() override;
/**
* In charge of initialising the primary Tensor as well as the staging
* tensor as required. It will only initialise a staging tensor if the
* Primary tensor is of type Device. For staging tensors it performs a
* mapDataIntoHostMemory which would perform immediately as opposed to
* on sequence eval/submission.
*/
void init() override;
/**
* Record runs the core actions to create the tensors. For device tensors
* it records a copyCommand to move the data from the staging tensor to the
* device tensor. The mapping for staging tensors happens in the init function
* not in the record function.
*/
void record() override;
/**
* Does not perform any preEval commands.
*/
virtual void preEval() override;
/**
* Performs a copy back into the main tensor to ensure that the data
* contained is the one that is now being stored in the GPU.
*/
virtual void postEval() override;
private:
// Never owned resources
std::vector<std::shared_ptr<Tensor>> mStagingTensors;
};
} // End namespace kp
namespace kp {
/**
Operation that copies the data from the first tensor to the rest of the tensors provided, using a record command for all the vectors. This operation does not own/manage the memory of the tensors passed to it. The operation must only receive tensors of type
*/

3
src/Tensor.cpp
View file

@ -100,6 +100,9 @@ Tensor::isInit()
void
Tensor::setData(const std::vector<float>& data)
{
if (data.size() != this->mData.size()) {
throw std::runtime_error("Kompute Tensor Cannot set data of different sizes");
}
this->mData = data;
}

28
src/include/kompute/Manager.hpp
View file

@ -6,6 +6,8 @@
#include "kompute/Sequence.hpp"
#include "kompute/operations/OpTensorCreate.hpp"
#define KP_DEFAULT_SESSION "DEFAULT"
namespace kp {
@ -15,7 +17,6 @@ namespace kp {
*/
class Manager
{
private:
public:
/**
Base constructor and default used which creates the base resources
@ -112,6 +113,31 @@ class Manager
this->evalOp<T>(tensors, KP_DEFAULT_SESSION, std::forward<TArgs>(params)...);
}
/**
* Function that simplifies the common workflow of tensor creation and
* initialization. It will take the constructor parameters for a Tensor
* and will will us it to create a new Tensor and then create it using
* the OpCreateTensor command.
*
* @param data The data to initialize the tensor with
* @param tensorType The type of tensor to initialize
* @returns Initialized Tensor with memory Syncd to GPU device
*/
std::shared_ptr<Tensor> buildTensor(
const std::vector<float>& data,
Tensor::TensorTypes tensorType = Tensor::TensorTypes::eDevice)
{
SPDLOG_DEBUG("Kompute Manager createInitTensor triggered");
SPDLOG_DEBUG("Kompute Manager creating new tensor shared ptr");
std::shared_ptr<Tensor> tensor = std::make_shared<Tensor>(kp::Tensor(data, tensorType));
this->evalOpDefault<OpTensorCreate>({tensor});
return tensor;
}
private:
// -------------- OPTIONALLY OWNED RESOURCES
std::shared_ptr<vk::Instance> mInstance = nullptr;

20
test/TestManager.cpp
View file

@ -107,3 +107,23 @@ TEST(TestManager, TestMultipleTensorsAtOnce) {
EXPECT_EQ(tensorOutput->data(), std::vector<float>({0, 4, 12}));
}
TEST(TestManager, TestCreateInitTensor) {
kp::Manager mgr;
std::shared_ptr<kp::Tensor> tensorA = mgr.buildTensor({0,1,2});
std::shared_ptr<kp::Tensor> tensorB = mgr.buildTensor({0,0,0});
mgr.evalOpDefault<kp::OpTensorCopy>({tensorA, tensorB});
mgr.evalOpDefault<kp::OpTensorSyncLocal>({tensorB});
EXPECT_EQ(tensorB->data(), std::vector<float>({0,1,2}));
std::shared_ptr<kp::Tensor> tensorC = mgr.buildTensor({0,0,0}, kp::Tensor::TensorTypes::eStaging);
mgr.evalOpDefault<kp::OpTensorCopy>({tensorA, tensorC});
EXPECT_EQ(tensorC->data(), std::vector<float>({0,1,2}));
}