thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Reformatted

Browse source
This commit is contained in:
Alejandro Saucedo 2020-08-22 12:31:11 +01:00
parent 181efc954b
commit 5bb9046b49
11 changed files with 148 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

160
src/Algorithm.cpp
View file

@ -9,7 +9,8 @@ Algorithm::Algorithm()
SPDLOG_DEBUG("Kompute Algorithm base constructor");
}
Algorithm::Algorithm(std::shared_ptr<vk::Device> device, std::shared_ptr<vk::CommandBuffer> commandBuffer)
Algorithm::Algorithm(std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer)
{
SPDLOG_DEBUG("Kompute Algorithm Constructor with device");
@ -28,8 +29,10 @@ Algorithm::~Algorithm()
}
}
void Algorithm::init(std::string shaderFilePath,
std::vector<std::shared_ptr<Tensor>> tensorParams) {
void
Algorithm::init(std::string shaderFilePath,
std::vector<std::shared_ptr<Tensor>> tensorParams)
{
SPDLOG_DEBUG("Kompute Algorithm init started");
// TODO: Move to util function
@ -38,77 +41,79 @@ void Algorithm::init(std::string shaderFilePath,
this->createPipeline();
}
void Algorithm::createParameters(std::vector<std::shared_ptr<Tensor>>& tensorParams) {
void
Algorithm::createParameters(std::vector<std::shared_ptr<Tensor>>& tensorParams)
{
SPDLOG_DEBUG("Kompute Algorithm createParameters started");
// TODO: Explore design for having multiple descriptor pool sizes
std::vector<vk::DescriptorPoolSize> descriptorPoolSizes = {
vk::DescriptorPoolSize(
vk::DescriptorType::eStorageBuffer,
1 // Descriptor count
)
};
vk::DescriptorPoolSize(vk::DescriptorType::eStorageBuffer,
1 // Descriptor count
)
};
// TODO: Explore design for having more than 1 set configurable
vk::DescriptorPoolCreateInfo descriptorPoolInfo(
vk::DescriptorPoolCreateFlags(),
1, // Max sets
static_cast<uint32_t>(descriptorPoolSizes.size()),
descriptorPoolSizes.data());
vk::DescriptorPoolCreateFlags(),
1, // Max sets
static_cast<uint32_t>(descriptorPoolSizes.size()),
descriptorPoolSizes.data());
SPDLOG_DEBUG("Kompute Algorithm creating descriptor pool");
this->mDescriptorPool = std::make_shared<vk::DescriptorPool>();
this->mDevice->createDescriptorPool(&descriptorPoolInfo, nullptr, this->mDescriptorPool.get());
this->mDevice->createDescriptorPool(
&descriptorPoolInfo, nullptr, this->mDescriptorPool.get());
std::vector<vk::DescriptorSetLayoutBinding> descriptorSetBindings;
// TODO: Explore allowing descriptor set bind index to be configurable by user to specify which tensors woudl go on each binding
// TODO: Explore allowing descriptor set bind index to be configurable by
// user to specify which tensors woudl go on each binding
for (size_t i = 0; i < tensorParams.size(); i++) {
descriptorSetBindings.push_back(
vk::DescriptorSetLayoutBinding(
i, // Binding index
vk::DescriptorType::eStorageBuffer,
1, // Descriptor count
vk::ShaderStageFlagBits::eCompute)
);
vk::DescriptorSetLayoutBinding(i, // Binding index
vk::DescriptorType::eStorageBuffer,
1, // Descriptor count
vk::ShaderStageFlagBits::eCompute));
}
// This is the component that is fed into the pipeline
vk::DescriptorSetLayoutCreateInfo descriptorSetLayoutInfo(
vk::DescriptorSetLayoutCreateFlags(),
static_cast<uint32_t>(descriptorSetBindings.size()),
descriptorSetBindings.data()
);
vk::DescriptorSetLayoutCreateFlags(),
static_cast<uint32_t>(descriptorSetBindings.size()),
descriptorSetBindings.data());
SPDLOG_DEBUG("Kompute Algorithm creating descriptor set layout");
// TODO: We createa signle descriptor set layout which would have to be extended if multiple set layouts to be supported
// TODO: We createa signle descriptor set layout which would have to be
// extended if multiple set layouts to be supported
this->mDescriptorSetLayout = std::make_shared<vk::DescriptorSetLayout>();
this->mDevice->createDescriptorSetLayout(&descriptorSetLayoutInfo, nullptr, this->mDescriptorSetLayout.get());
this->mDevice->createDescriptorSetLayout(
&descriptorSetLayoutInfo, nullptr, this->mDescriptorSetLayout.get());
vk::DescriptorSetAllocateInfo descriptorSetAllocateInfo(
*this->mDescriptorPool,
1, // Descriptor set layout count
this->mDescriptorSetLayout.get());
*this->mDescriptorPool,
1, // Descriptor set layout count
this->mDescriptorSetLayout.get());
SPDLOG_DEBUG("Kompute Algorithm allocating descriptor sets");
this->mDescriptorSet = std::make_shared<vk::DescriptorSet>();
this->mDevice->allocateDescriptorSets(&descriptorSetAllocateInfo, this->mDescriptorSet.get());
this->mDevice->allocateDescriptorSets(&descriptorSetAllocateInfo,
this->mDescriptorSet.get());
std::vector<vk::WriteDescriptorSet> computeWriteDescriptorSets;
for (size_t i = 0; i < tensorParams.size(); i++) {
vk::DescriptorBufferInfo descriptorBufferInfo =
tensorParams[i]->constructDescriptorBufferInfo();
tensorParams[i]->constructDescriptorBufferInfo();
// TODO: Explore design exposing the destination array element
computeWriteDescriptorSets.push_back(
vk::WriteDescriptorSet(
*this->mDescriptorSet,
i, // Destination binding
0, // Destination array element
1, // Descriptor count
vk::DescriptorType::eStorageBuffer,
nullptr, // Descriptor image info
&descriptorBufferInfo));
vk::WriteDescriptorSet(*this->mDescriptorSet,
i, // Destination binding
0, // Destination array element
1, // Descriptor count
vk::DescriptorType::eStorageBuffer,
nullptr, // Descriptor image info
&descriptorBufferInfo));
}
SPDLOG_DEBUG("Kompute Algorithm updating descriptor sets");
@ -117,11 +122,13 @@ void Algorithm::createParameters(std::vector<std::shared_ptr<Tensor>>& tensorPar
SPDLOG_DEBUG("Kompue Algorithm successfully run init");
}
void Algorithm::createShaderModule(std::string shaderFilePath) {
void
Algorithm::createShaderModule(std::string shaderFilePath)
{
SPDLOG_DEBUG("Kompute Algorithm createShaderModule started");
std::ifstream fileStream(
shaderFilePath, std::ios::binary | std::ios::in | std::ios::ate);
std::ifstream fileStream(shaderFilePath,
std::ios::binary | std::ios::in | std::ios::ate);
size_t shaderFileSize = fileStream.tellg();
fileStream.seekg(0, std::ios::beg);
@ -130,64 +137,79 @@ void Algorithm::createShaderModule(std::string shaderFilePath) {
fileStream.close();
vk::ShaderModuleCreateInfo shaderModuleInfo(
vk::ShaderModuleCreateFlags(),
shaderFileSize,
(uint32_t*)shaderFileData);
vk::ShaderModuleCreateFlags(), shaderFileSize, (uint32_t*)shaderFileData);
SPDLOG_DEBUG("Kompute Algorithm Creating shader module. ShaderFileSize: {}", shaderFileSize);
SPDLOG_DEBUG("Kompute Algorithm Creating shader module. ShaderFileSize: {}",
shaderFileSize);
this->mFreeShaderModule = true;
this->mShaderModule = std::make_shared<vk::ShaderModule>();
this->mDevice->createShaderModule(
&shaderModuleInfo,
nullptr,
this->mShaderModule.get());
&shaderModuleInfo, nullptr, this->mShaderModule.get());
SPDLOG_DEBUG("Kompute Algorithm create shader module success");
}
void Algorithm::createPipeline() {
void
Algorithm::createPipeline()
{
SPDLOG_DEBUG("Kompute Algorithm calling create Pipeline");
// TODO: Explore design for supporting multiple sets
vk::PipelineLayoutCreateInfo pipelineLayoutInfo(
vk::PipelineLayoutCreateFlags(),
1, // Set layout count
this->mDescriptorSetLayout.get());
vk::PipelineLayoutCreateFlags(),
1, // Set layout count
this->mDescriptorSetLayout.get());
this->mPipelineLayout = std::make_shared<vk::PipelineLayout>();
this->mDevice->createPipelineLayout(&pipelineLayoutInfo, nullptr, this->mPipelineLayout.get());
this->mDevice->createPipelineLayout(
&pipelineLayoutInfo, nullptr, this->mPipelineLayout.get());
vk::PipelineShaderStageCreateInfo shaderStage(vk::PipelineShaderStageCreateFlags(), vk::ShaderStageFlagBits::eCompute, *this->mShaderModule, "main", nullptr);
vk::PipelineShaderStageCreateInfo shaderStage(
vk::PipelineShaderStageCreateFlags(),
vk::ShaderStageFlagBits::eCompute,
*this->mShaderModule,
"main",
nullptr);
vk::ComputePipelineCreateInfo pipelineInfo(vk::PipelineCreateFlags(), shaderStage, *this->mPipelineLayout, vk::Pipeline(), 0);
vk::ComputePipelineCreateInfo pipelineInfo(vk::PipelineCreateFlags(),
shaderStage,
*this->mPipelineLayout,
vk::Pipeline(),
0);
// TODO: Confirm what the best structure is with pipeline cache
this->mFreePipelineCache = true;
vk::PipelineCacheCreateInfo pipelineCacheInfo = vk::PipelineCacheCreateInfo();
vk::PipelineCacheCreateInfo pipelineCacheInfo =
vk::PipelineCacheCreateInfo();
this->mPipelineCache = std::make_shared<vk::PipelineCache>();
this->mDevice->createPipelineCache(&pipelineCacheInfo, nullptr, this->mPipelineCache.get());
this->mDevice->createPipelineCache(
&pipelineCacheInfo, nullptr, this->mPipelineCache.get());
vk::ResultValue<vk::Pipeline> pipelineResult = this->mDevice->createComputePipeline(*this->mPipelineCache, pipelineInfo);
vk::ResultValue<vk::Pipeline> pipelineResult =
this->mDevice->createComputePipeline(*this->mPipelineCache, pipelineInfo);
if (pipelineResult.result != vk::Result::eSuccess) {
throw std::runtime_error("Failed to create pipeline result: " + vk::to_string(pipelineResult.result));
throw std::runtime_error("Failed to create pipeline result: " +
vk::to_string(pipelineResult.result));
}
this->mFreePipeline = true;
this->mPipeline = std::make_shared<vk::Pipeline>(pipelineResult.value);
}
void Algorithm::recordDispatch(uint32_t x, uint32_t y, uint32_t z) {
void
Algorithm::recordDispatch(uint32_t x, uint32_t y, uint32_t z)
{
SPDLOG_DEBUG("Kompute Algorithm calling record dispatch");
this->mCommandBuffer->bindPipeline(vk::PipelineBindPoint::eCompute, *this->mPipeline);
this->mCommandBuffer->bindPipeline(vk::PipelineBindPoint::eCompute,
*this->mPipeline);
this->mCommandBuffer->bindDescriptorSets(
vk::PipelineBindPoint::eCompute,
*this->mPipelineLayout,
0, // First set
*this->mDescriptorSet,
nullptr // Dispatcher
this->mCommandBuffer->bindDescriptorSets(vk::PipelineBindPoint::eCompute,
*this->mPipelineLayout,
0, // First set
*this->mDescriptorSet,
nullptr // Dispatcher
);
this->mCommandBuffer->dispatch(x, y, z);

5
src/Algorithm.hpp
View file

@ -19,7 +19,8 @@ class Algorithm
public:
Algorithm();
Algorithm(std::shared_ptr<vk::Device> device, std::shared_ptr<vk::CommandBuffer> commandBuffer);
Algorithm(std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer);
// TODO: Add specialisation data
// TODO: Explore other ways of passing shader (ie raw bytes)
@ -31,7 +32,7 @@ class Algorithm
// Record commands
void recordDispatch(uint32_t x, uint32_t y, uint32_t z);
private:
private:
// Shared resources
std::shared_ptr<vk::Device> mDevice;
std::shared_ptr<vk::CommandBuffer> mCommandBuffer;

10
src/OpBase.hpp
View file

@ -20,9 +20,7 @@ class OpBase
{
private:
public:
OpBase() {
SPDLOG_DEBUG("Compute OpBase base constructor");
}
OpBase() { SPDLOG_DEBUG("Compute OpBase base constructor"); }
OpBase(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
@ -44,11 +42,7 @@ class OpBase
virtual void record() { SPDLOG_DEBUG("Kompute OpBase record called"); }
virtual void postSubmit()
{
SPDLOG_DEBUG("Kompute OpBase init called");
}
virtual void postSubmit() { SPDLOG_DEBUG("Kompute OpBase init called"); }
protected:
std::shared_ptr<vk::PhysicalDevice> mPhysicalDevice;

4
src/OpCreateTensor.cpp
View file

@ -65,10 +65,10 @@ OpCreateTensor::record()
}
}
void OpCreateTensor::postSubmit()
void
OpCreateTensor::postSubmit()
{
SPDLOG_DEBUG("Kompute OpCreateTensor postSubmit called");
}
}

32
src/OpMult.cpp
View file

@ -46,23 +46,36 @@ OpMult::init(std::vector<std::shared_ptr<Tensor>> tensors)
this->mTensorOutput = tensors[2];
// TODO: Explore adding a validate function
if (!(this->mTensorLHS->isInit() && this->mTensorRHS->isInit() && this->mTensorOutput->isInit())) {
throw std::runtime_error("Kompute OpMult all tensor parameters must be initialised. LHS: " + std::to_string(this->mTensorLHS->isInit()) + " RHS: " + std::to_string(this->mTensorRHS->isInit()) + " Output: " + std::to_string(this->mTensorOutput->isInit()));
if (!(this->mTensorLHS->isInit() && this->mTensorRHS->isInit() &&
this->mTensorOutput->isInit())) {
throw std::runtime_error(
"Kompute OpMult all tensor parameters must be initialised. LHS: " +
std::to_string(this->mTensorLHS->isInit()) +
" RHS: " + std::to_string(this->mTensorRHS->isInit()) +
" Output: " + std::to_string(this->mTensorOutput->isInit()));
}
// TODO: Explore use-cases where tensors shouldn't be the same size, and how to deal with those situations
if (!(this->mTensorLHS->size() == this->mTensorRHS->size() && this->mTensorRHS->size() == this->mTensorOutput->size())) {
throw std::runtime_error("Kompute OpMult all tensor parameters must be the same size LHS: " + std::to_string(this->mTensorLHS->size()) + " RHS: " + std::to_string(this->mTensorRHS->size()) + " Output: " + std::to_string(this->mTensorOutput->size()));
// TODO: Explore use-cases where tensors shouldn't be the same size, and how
// to deal with those situations
if (!(this->mTensorLHS->size() == this->mTensorRHS->size() &&
this->mTensorRHS->size() == this->mTensorOutput->size())) {
throw std::runtime_error(
"Kompute OpMult all tensor parameters must be the same size LHS: " +
std::to_string(this->mTensorLHS->size()) +
" RHS: " + std::to_string(this->mTensorRHS->size()) +
" Output: " + std::to_string(this->mTensorOutput->size()));
}
this->mTensorOutputStaging = std::make_shared<Tensor>(
this->mTensorOutput->data(), Tensor::TensorTypes::eStaging);
this->mTensorOutputStaging->init(this->mPhysicalDevice, this->mDevice, this->mCommandBuffer, this->mTensorOutput->data());
this->mTensorOutputStaging->init(this->mPhysicalDevice,
this->mDevice,
this->mCommandBuffer,
this->mTensorOutput->data());
// TODO: Make this path configurable
this->mAlgorithm->init(
"shaders/glsl/opmult.comp.spv", tensors);
this->mAlgorithm->init("shaders/glsl/opmult.comp.spv", tensors);
}
void
@ -75,7 +88,8 @@ OpMult::record()
this->mTensorOutputStaging->recordCopyFrom(this->mTensorOutput);
}
void OpMult::postSubmit()
void
OpMult::postSubmit()
{
SPDLOG_DEBUG("Kompute OpCreateTensor postSubmit called");

2
src/OpMult.hpp
View file

@ -10,8 +10,8 @@
#include <spdlog/spdlog.h>
#include "Tensor.hpp"
#include "Algorithm.hpp"
#include "Tensor.hpp"
#include "OpBase.hpp"

2
src/Parameter.hpp
View file

@ -28,7 +28,7 @@ class Algorithm
~Algorithm();
private:
private:
// Shared resources
std::shared_ptr<vk::Device> mDevice;

3
src/Sequence.cpp
View file

@ -120,7 +120,8 @@ Sequence::eval()
this->end();
}
// TODO: Explore whether moving postSubmit calls to a separate sequence function that is explicitly called by the manager
// TODO: Explore whether moving postSubmit calls to a separate sequence
// function that is explicitly called by the manager
for (size_t i = 0; i < this->mOperations.size(); i++) {
this->mOperations[i]->postSubmit();
}

33
src/Tensor.cpp
View file

@ -105,7 +105,9 @@ Tensor::isInit()
return this->mIsInit;
}
void Tensor::setData(const std::vector<uint32_t>& data) {
void
Tensor::setData(const std::vector<uint32_t>& data)
{
this->mData = data;
}
@ -135,36 +137,43 @@ Tensor::recordCopyFrom(std::shared_ptr<Tensor> copyFromTensor)
}
// TODO: Explore if this function should be here or expose buffer
vk::DescriptorBufferInfo Tensor::constructDescriptorBufferInfo() {
vk::DescriptorBufferInfo
Tensor::constructDescriptorBufferInfo()
{
vk::DeviceSize bufferSize = this->memorySize();
return vk::DescriptorBufferInfo(
*this->mBuffer,
0, // offset
bufferSize
);
return vk::DescriptorBufferInfo(*this->mBuffer,
0, // offset
bufferSize);
}
void Tensor::copyDataFromHostBuffer() {
void
Tensor::copyDataFromHostBuffer()
{
SPDLOG_DEBUG("Kompute Tensor copying data from host buffer");
if (this->mTensorType != TensorTypes::eStaging) {
spdlog::warn("Copying tensor data manually to DEVICE buffer instead of using record GPU command");
spdlog::warn("Copying tensor data manually to DEVICE buffer instead of "
"using record GPU command");
}
vk::DeviceSize bufferSize = this->memorySize();
void* mapped = this->mDevice->mapMemory(*this->mMemory, 0, bufferSize, vk::MemoryMapFlags());
void* mapped = this->mDevice->mapMemory(
*this->mMemory, 0, bufferSize, vk::MemoryMapFlags());
vk::MappedMemoryRange mappedMemoryRange(*this->mMemory, 0, bufferSize);
this->mDevice->invalidateMappedMemoryRanges(mappedMemoryRange);
memcpy(this->mData.data(), mapped, bufferSize);
this->mDevice->unmapMemory(*this->mMemory);
}
void Tensor::copyDataToHostBuffer() {
void
Tensor::copyDataToHostBuffer()
{
SPDLOG_DEBUG("Kompute Tensor copying data to buffer");
if (this->mTensorType != TensorTypes::eStaging) {
spdlog::warn("Copying tensor data manually to DEVICE buffer instead of using record GPU command");
spdlog::warn("Copying tensor data manually to DEVICE buffer instead of "
"using record GPU command");
}
vk::DeviceSize bufferSize = this->memorySize();

2
src/Tensor.hpp
View file

@ -52,7 +52,7 @@ class Tensor
// Record functions
void recordCopyFrom(std::shared_ptr<Tensor> copyFromTensor);
// TODO: Add memory buffer barrier capabilities
//void recordBufferMemoryBarrier();
// void recordBufferMemoryBarrier();
// Util functions
vk::DescriptorBufferInfo constructDescriptorBufferInfo();

2
src/main.cpp
View file

@ -637,7 +637,7 @@ main()
// TODO: Add capabilities for just output tensor types
spdlog::info("Creating output tensor");
std::shared_ptr<kp::Tensor> tensorOutput{ new kp::Tensor(
{ 0.0, 0.0, 0.0 }) };
{ 0.0, 0.0, 0.0 }) };
mgr.evalOp<kp::OpCreateTensor>({ tensorOutput });
spdlog::info("OpCreateTensor success for tensors");