thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Reformatted

Browse source
This commit is contained in:
Alejandro Saucedo 2020-11-01 20:56:03 +00:00
parent 3036cbd95f
commit 1f614a87e4
11 changed files with 125 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

5
single_include/kompute/Kompute.hpp
View file

@ -1315,8 +1315,9 @@ class Manager
* @param queueIndex The queue to use from the available queues
* @return Weak pointer to the manager owned sequence resource
*/
std::shared_ptr<Sequence> createManagedSequence(std::string sequenceName = "",
uint32_t queueIndex = 0);
std::shared_ptr<Sequence> createManagedSequence(
std::string sequenceName = "",
uint32_t queueIndex = 0);
/**
* Function that evaluates operation against named sequence.

24
src/Algorithm.cpp
View file

@ -34,7 +34,9 @@ Algorithm::~Algorithm()
SPDLOG_ERROR("Kompute Algorithm Error requested to destroy "
"pipeline but it is null");
}
this->mDevice->destroy(*this->mPipeline, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->destroy(
*this->mPipeline,
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
}
if (this->mFreePipelineCache) {
@ -43,7 +45,9 @@ Algorithm::~Algorithm()
SPDLOG_ERROR("Kompute Algorithm Error requested to destroy "
"pipeline cache but it is null");
}
this->mDevice->destroy(*this->mPipelineCache, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->destroy(
*this->mPipelineCache,
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
}
if (this->mFreePipelineLayout) {
@ -52,7 +56,9 @@ Algorithm::~Algorithm()
SPDLOG_ERROR("Kompute Algorithm Error requested to destroy "
"pipeline layout but it is null");
}
this->mDevice->destroy(*this->mPipelineLayout, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->destroy(
*this->mPipelineLayout,
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
}
if (this->mFreeShaderModule) {
@ -61,7 +67,9 @@ Algorithm::~Algorithm()
SPDLOG_ERROR("Kompute Algorithm Error requested to destroy shader "
"module but it is null");
}
this->mDevice->destroy(*this->mShaderModule, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->destroy(
*this->mShaderModule,
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
}
if (this->mFreeDescriptorSet) {
@ -80,7 +88,9 @@ Algorithm::~Algorithm()
SPDLOG_ERROR("Kompute Algorithm Error requested to destroy "
"descriptor set layout but it is null");
}
this->mDevice->destroy(*this->mDescriptorSetLayout, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->destroy(
*this->mDescriptorSetLayout,
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
}
if (this->mFreeDescriptorPool) {
@ -89,7 +99,9 @@ Algorithm::~Algorithm()
SPDLOG_ERROR("Kompute Algorithm Error requested to destroy "
"descriptor pool but it is null");
}
this->mDevice->destroy(*this->mDescriptorPool, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->destroy(
*this->mDescriptorPool,
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
}
}

12
src/Manager.cpp
View file

@ -59,8 +59,10 @@ Manager::~Manager()
}
if (this->mManagedSequences.size()) {
SPDLOG_DEBUG("Kompute Manager explicitly running destructor for managed sequences");
for (const std::pair<std::string, std::shared_ptr<Sequence>> &sqPair : this->mManagedSequences) {
SPDLOG_DEBUG("Kompute Manager explicitly running destructor for "
"managed sequences");
for (const std::pair<std::string, std::shared_ptr<Sequence>>& sqPair :
this->mManagedSequences) {
sqPair.second->~Sequence();
}
this->mManagedSequences.clear();
@ -68,7 +70,8 @@ Manager::~Manager()
if (this->mFreeDevice) {
SPDLOG_INFO("Destroying device");
this->mDevice->destroy((vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->destroy(
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
SPDLOG_DEBUG("Kompute Manager Destroyed Device");
}
@ -89,7 +92,8 @@ Manager::~Manager()
#endif
if (this->mFreeInstance) {
this->mInstance->destroy((vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mInstance->destroy(
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
SPDLOG_DEBUG("Kompute Manager Destroyed Instance");
}
}

88
src/OpAlgoBase.cpp
View file

@ -10,13 +10,14 @@ OpAlgoBase::OpAlgoBase()
}
OpAlgoBase::OpAlgoBase(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer,
std::vector<std::shared_ptr<Tensor>>& tensors,
KomputeWorkgroup komputeWorkgroup)
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer,
std::vector<std::shared_ptr<Tensor>>& tensors,
KomputeWorkgroup komputeWorkgroup)
: OpBase(physicalDevice, device, commandBuffer, tensors, false)
{
SPDLOG_DEBUG("Kompute OpAlgoBase constructor with params numTensors: {}", tensors.size());
SPDLOG_DEBUG("Kompute OpAlgoBase constructor with params numTensors: {}",
tensors.size());
// The dispatch size is set up based on either explicitly provided template
// parameters or by default it would take the shape and size of the tensors
@ -29,38 +30,42 @@ OpAlgoBase::OpAlgoBase(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
komputeWorkgroup.z > 0 ? komputeWorkgroup.z : 1
};
} else {
this->mKomputeWorkgroup = {tensors[0]->size(), 1, 1};
this->mKomputeWorkgroup = { tensors[0]->size(), 1, 1 };
}
SPDLOG_INFO("Kompute OpAlgoBase dispatch size X: {}, Y: {}, Z: {}",
this->mKomputeWorkgroup.x,
this->mKomputeWorkgroup.y,
this->mKomputeWorkgroup.z);
this->mKomputeWorkgroup.x,
this->mKomputeWorkgroup.y,
this->mKomputeWorkgroup.z);
this->mAlgorithm = std::make_shared<Algorithm>(device, commandBuffer);
}
OpAlgoBase::OpAlgoBase(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer,
std::vector<std::shared_ptr<Tensor>>& tensors,
std::string shaderFilePath,
KomputeWorkgroup komputeWorkgroup)
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer,
std::vector<std::shared_ptr<Tensor>>& tensors,
std::string shaderFilePath,
KomputeWorkgroup komputeWorkgroup)
: OpAlgoBase(physicalDevice, device, commandBuffer, tensors, komputeWorkgroup)
{
SPDLOG_DEBUG("Kompute OpAlgoBase shaderFilePath constructo with shaderfile path: {}", shaderFilePath);
SPDLOG_DEBUG(
"Kompute OpAlgoBase shaderFilePath constructo with shaderfile path: {}",
shaderFilePath);
this->mShaderFilePath = shaderFilePath;
}
OpAlgoBase::OpAlgoBase(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer,
std::vector<std::shared_ptr<Tensor>>& tensors,
const std::vector<char>& shaderDataRaw,
KomputeWorkgroup komputeWorkgroup)
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer,
std::vector<std::shared_ptr<Tensor>>& tensors,
const std::vector<char>& shaderDataRaw,
KomputeWorkgroup komputeWorkgroup)
: OpAlgoBase(physicalDevice, device, commandBuffer, tensors, komputeWorkgroup)
{
SPDLOG_DEBUG("Kompute OpAlgoBase shaderFilePath constructo with shader raw data length: {}", shaderDataRaw.size());
SPDLOG_DEBUG("Kompute OpAlgoBase shaderFilePath constructo with shader raw "
"data length: {}",
shaderDataRaw.size());
this->mShaderDataRaw = shaderDataRaw;
}
@ -78,11 +83,13 @@ OpAlgoBase::init()
if (this->mTensors.size() < 1) {
throw std::runtime_error(
"Kompute OpAlgoBase called with less than 1 tensor");
}
}
for (std::shared_ptr<Tensor> tensor : this->mTensors) {
if(!tensor->isInit()) {
throw std::runtime_error("Kompute OpAlgoBase validation failed; all tensor parameters must be initialised.");
if (!tensor->isInit()) {
throw std::runtime_error(
"Kompute OpAlgoBase validation failed; all tensor parameters "
"must be initialised.");
}
}
@ -110,7 +117,9 @@ OpAlgoBase::record()
vk::PipelineStageFlagBits::eComputeShader);
}
this->mAlgorithm->recordDispatch(this->mKomputeWorkgroup.x, this->mKomputeWorkgroup.y, this->mKomputeWorkgroup.z);
this->mAlgorithm->recordDispatch(this->mKomputeWorkgroup.x,
this->mKomputeWorkgroup.y,
this->mKomputeWorkgroup.z);
}
void
@ -125,17 +134,19 @@ OpAlgoBase::postEval()
SPDLOG_DEBUG("Kompute OpAlgoBase postSubmit called");
}
std::vector<char> OpAlgoBase::fetchSpirvBinaryData()
std::vector<char>
OpAlgoBase::fetchSpirvBinaryData()
{
SPDLOG_WARN(
"Kompute OpAlgoBase Running shaders directly from spirv file");
SPDLOG_WARN("Kompute OpAlgoBase Running shaders directly from spirv file");
if (this->mShaderFilePath.size()) {
std::ifstream fileStream(this->mShaderFilePath,
std::ios::binary | std::ios::in | std::ios::ate);
std::ios::binary | std::ios::in |
std::ios::ate);
if (!fileStream.good()) {
throw std::runtime_error("Error reading file: " + this->mShaderFilePath);
throw std::runtime_error("Error reading file: " +
this->mShaderFilePath);
}
size_t shaderFileSize = fileStream.tellg();
@ -144,19 +155,16 @@ std::vector<char> OpAlgoBase::fetchSpirvBinaryData()
fileStream.read(shaderDataRaw, shaderFileSize);
fileStream.close();
SPDLOG_WARN(
"Kompute OpAlgoBase fetched {} bytes", shaderFileSize);
SPDLOG_WARN("Kompute OpAlgoBase fetched {} bytes", shaderFileSize);
return std::vector<char>(shaderDataRaw,
shaderDataRaw + shaderFileSize);
}
else if (this->mShaderDataRaw.size()) {
return std::vector<char>(shaderDataRaw, shaderDataRaw + shaderFileSize);
} else if (this->mShaderDataRaw.size()) {
return this->mShaderDataRaw;
}
else {
throw std::runtime_error("Kompute OpAlgoBase Error reached fetchSpirvBinaryData but neither filepath nor data provided");
} else {
throw std::runtime_error(
"Kompute OpAlgoBase Error reached fetchSpirvBinaryData but neither "
"filepath nor data provided");
}
}
}

38
src/OpAlgoLhsRhsOut.cpp
View file

@ -9,13 +9,14 @@ OpAlgoLhsRhsOut::OpAlgoLhsRhsOut()
SPDLOG_DEBUG("Kompute OpAlgoLhsRhsOut constructor base");
}
OpAlgoLhsRhsOut::OpAlgoLhsRhsOut(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer,
std::vector<std::shared_ptr<Tensor>> tensors,
KomputeWorkgroup komputeWorkgroup)
OpAlgoLhsRhsOut::OpAlgoLhsRhsOut(
std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer,
std::vector<std::shared_ptr<Tensor>> tensors,
KomputeWorkgroup komputeWorkgroup)
// The inheritance is initialised with the copyOutputData to false given that
// this depencendant class handles the transfer of data via staging buffers in
// this depencendant class handles the transfer of data via staging buffers in
// a granular way.
: OpAlgoBase(physicalDevice, device, commandBuffer, tensors, komputeWorkgroup)
{
@ -36,18 +37,19 @@ OpAlgoLhsRhsOut::init()
throw std::runtime_error(
"Kompute OpAlgoLhsRhsOut called with less than 1 tensor");
} else if (this->mTensors.size() > 3) {
SPDLOG_WARN("Kompute OpAlgoLhsRhsOut called with more than 3 this->mTensors");
SPDLOG_WARN(
"Kompute OpAlgoLhsRhsOut called with more than 3 this->mTensors");
}
this->mTensorLHS = this->mTensors[0];
this->mTensorRHS = this->mTensors[1];
this->mTensorOutput = this->mTensors[2];
if (!(this->mTensorLHS->isInit() && this->mTensorRHS->isInit() &&
this->mTensorOutput->isInit())) {
throw std::runtime_error(
"Kompute OpAlgoLhsRhsOut all tensor parameters must be initialised. LHS: " +
"Kompute OpAlgoLhsRhsOut all tensor parameters must be initialised. "
"LHS: " +
std::to_string(this->mTensorLHS->isInit()) +
" RHS: " + std::to_string(this->mTensorRHS->isInit()) +
" Output: " + std::to_string(this->mTensorOutput->isInit()));
@ -56,7 +58,8 @@ OpAlgoLhsRhsOut::init()
if (!(this->mTensorLHS->size() == this->mTensorRHS->size() &&
this->mTensorRHS->size() == this->mTensorOutput->size())) {
throw std::runtime_error(
"Kompute OpAlgoLhsRhsOut all tensor parameters must be the same size LHS: " +
"Kompute OpAlgoLhsRhsOut all tensor parameters must be the same size "
"LHS: " +
std::to_string(this->mTensorLHS->size()) +
" RHS: " + std::to_string(this->mTensorRHS->size()) +
" Output: " + std::to_string(this->mTensorOutput->size()));
@ -65,8 +68,7 @@ OpAlgoLhsRhsOut::init()
this->mTensorOutputStaging = std::make_shared<Tensor>(
this->mTensorOutput->data(), Tensor::TensorTypes::eStaging);
this->mTensorOutputStaging->init(
this->mPhysicalDevice, this->mDevice);
this->mTensorOutputStaging->init(this->mPhysicalDevice, this->mDevice);
SPDLOG_DEBUG("Kompute OpAlgoLhsRhsOut fetching spirv data");
@ -96,10 +98,9 @@ OpAlgoLhsRhsOut::record()
vk::PipelineStageFlagBits::eHost,
vk::PipelineStageFlagBits::eComputeShader);
this->mAlgorithm->recordDispatch(
this->mKomputeWorkgroup.x,
this->mKomputeWorkgroup.y,
this->mKomputeWorkgroup.z);
this->mAlgorithm->recordDispatch(this->mKomputeWorkgroup.x,
this->mKomputeWorkgroup.y,
this->mKomputeWorkgroup.z);
// Barrier to ensure the shader code is executed before buffer read
this->mTensorOutput->recordBufferMemoryBarrier(
@ -110,9 +111,7 @@ OpAlgoLhsRhsOut::record()
vk::PipelineStageFlagBits::eTransfer);
this->mTensorOutputStaging->recordCopyFrom(
this->mCommandBuffer,
this->mTensorOutput,
true);
this->mCommandBuffer, this->mTensorOutput, true);
}
void
@ -126,4 +125,3 @@ OpAlgoLhsRhsOut::postEval()
}
}

10
src/Sequence.cpp
View file

@ -28,7 +28,8 @@ Sequence::~Sequence()
SPDLOG_DEBUG("Kompute Sequence Destructor started");
if (!this->mIsInit) {
SPDLOG_WARN("Kompute Sequence destructor called but sequence is not initialized.");
SPDLOG_WARN("Kompute Sequence destructor called but sequence is not "
"initialized.");
return;
}
@ -60,7 +61,9 @@ Sequence::~Sequence()
this->mIsInit = false;
return;
}
this->mDevice->destroy(*this->mCommandPool, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->destroy(
*this->mCommandPool,
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
SPDLOG_DEBUG("Kompute Sequence Destroyed CommandPool");
}
@ -196,7 +199,8 @@ Sequence::evalAwait(uint64_t waitFor)
vk::Result result =
this->mDevice->waitForFences(1, &this->mFence, VK_TRUE, waitFor);
this->mDevice->destroy(this->mFence, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->destroy(
this->mFence, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mIsRunning = false;

13
src/Tensor.cpp
View file

@ -12,8 +12,9 @@ Tensor::Tensor()
Tensor::Tensor(const std::vector<float>& data, TensorTypes tensorType)
{
#if DEBUG
SPDLOG_DEBUG(
"Kompute Tensor constructor data length: {}, and type: {}", data.size(), tensorType);
SPDLOG_DEBUG("Kompute Tensor constructor data length: {}, and type: {}",
data.size(),
tensorType);
#endif
this->mData = data;
@ -350,7 +351,9 @@ Tensor::freeMemoryDestroyGPUResources()
"Kompose Tensor expected to free buffer but got null buffer");
} else {
SPDLOG_DEBUG("Kompose Tensor destroying buffer");
this->mDevice->destroy(*this->mBuffer, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->destroy(
*this->mBuffer,
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mBuffer = nullptr;
}
}
@ -361,7 +364,9 @@ Tensor::freeMemoryDestroyGPUResources()
"Kompose Tensor expected to free buffer but got null memory");
} else {
SPDLOG_DEBUG("Kompose Tensor freeing memory");
this->mDevice->freeMemory(*this->mMemory, (vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice->freeMemory(
*this->mMemory,
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
this->mDevice = nullptr;
}
}

5
src/include/kompute/Manager.hpp
View file

@ -77,8 +77,9 @@ class Manager
* @param queueIndex The queue to use from the available queues
* @return Weak pointer to the manager owned sequence resource
*/
std::shared_ptr<Sequence> createManagedSequence(std::string sequenceName = "",
uint32_t queueIndex = 0);
std::shared_ptr<Sequence> createManagedSequence(
std::string sequenceName = "",
uint32_t queueIndex = 0);
/**
* Function that evaluates operation against named sequence.

10
test/TestLogisticRegression.cpp
View file

@ -31,8 +31,7 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)
{
kp::Manager mgr;
std::shared_ptr<kp::Sequence> sqTensor =
mgr.createManagedSequence();
std::shared_ptr<kp::Sequence> sqTensor = mgr.createManagedSequence();
sqTensor->begin();
sqTensor->record<kp::OpTensorCreate>(params);
@ -76,7 +75,7 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)
EXPECT_LT(bIn->data()[0], 0.0);
EXPECT_LT(bIn->data()[0], 0.0);
//SPDLOG_WARN("Result wIn: {}, bIn: {}, loss: {}",
// SPDLOG_WARN("Result wIn: {}, bIn: {}, loss: {}",
// wIn->data(),
// bIn->data(),
// lOut->data());
@ -114,8 +113,7 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegressionManualCopy)
{
kp::Manager mgr;
std::shared_ptr<kp::Sequence> sqTensor =
mgr.createManagedSequence();
std::shared_ptr<kp::Sequence> sqTensor = mgr.createManagedSequence();
sqTensor->begin();
sqTensor->record<kp::OpTensorCreate>(params);
@ -158,7 +156,7 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegressionManualCopy)
EXPECT_GT(wIn->data()[1], 1.0);
EXPECT_LT(bIn->data()[0], 0.0);
//SPDLOG_WARN("Result wIn: {}, bIn: {}, loss: {}",
// SPDLOG_WARN("Result wIn: {}, bIn: {}, loss: {}",
// wIn->data(),
// bIn->data(),
// lOut->data());

15
test/TestMultipleAlgoExecutions.cpp
View file

@ -70,20 +70,20 @@ TEST(TestMultipleAlgoExecutions, MultipleCmdBufRecords)
// Then perform the computations
sq->begin();
sq->record<kp::OpAlgoBase>(
{ tensorA }, std::vector<char>(shader.begin(), shader.end()));
sq->record<kp::OpAlgoBase>({ tensorA },
std::vector<char>(shader.begin(), shader.end()));
sq->end();
sq->eval();
sq->begin();
sq->record<kp::OpAlgoBase>(
{ tensorA }, std::vector<char>(shader.begin(), shader.end()));
sq->record<kp::OpAlgoBase>({ tensorA },
std::vector<char>(shader.begin(), shader.end()));
sq->end();
sq->eval();
sq->begin();
sq->record<kp::OpAlgoBase>(
{ tensorA }, std::vector<char>(shader.begin(), shader.end()));
sq->record<kp::OpAlgoBase>({ tensorA },
std::vector<char>(shader.begin(), shader.end()));
sq->end();
sq->eval();
@ -112,7 +112,7 @@ TEST(TestMultipleAlgoExecutions, MultipleSequences)
})");
{
std::shared_ptr<kp::Sequence> sq =
std::shared_ptr<kp::Sequence> sq =
mgr.getOrCreateManagedSequence("newSequence");
sq->begin();
@ -211,7 +211,6 @@ TEST(TestMultipleAlgoExecutions, SingleRecordMultipleEval)
sq->eval();
}
{
std::shared_ptr<kp::Sequence> sq =
mgr.getOrCreateManagedSequence("newSequence3");

1
test/TestOpAlgoLoopsPassingData.cpp
View file

@ -30,7 +30,6 @@ TEST(TestProcessingIterations, IterateThroughMultipleSumAndCopies)
}
)");
{
std::shared_ptr<kp::Sequence> sq =
mgr.getOrCreateManagedSequence("default");