thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Initil implementation

Browse source 
Signed-off-by: Alejandro Saucedo <axsauze@gmail.com>
This commit is contained in:
Alejandro Saucedo 2021-09-12 09:37:50 +01:00
parent 44e4ff6978
commit 860fda9fb5
6 changed files with 247 additions and 110 deletions
Download patch file Download diff file Expand all files Collapse all files

120
single_include/kompute/Kompute.hpp
View file

@ -1094,12 +1094,30 @@ class Algorithm
* these can be modified but all new values must have the same vector size
* as this initial value.
*/
template<typename S = float, typename P = float>
Algorithm(std::shared_ptr<vk::Device> device,
const std::vector<std::shared_ptr<Tensor>>& tensors = {},
const std::vector<uint32_t>& spirv = {},
const Workgroup& workgroup = {},
const Constants& specializationConstants = {},
const Constants& pushConstants = {});
const std::vector<S>& specializationConstants = {},
const std::vector<P>& pushConstants = {})
{
KP_LOG_DEBUG("Kompute Algorithm Constructor with device");
this->mDevice = device;
if (tensors.size() && spirv.size()) {
KP_LOG_INFO("Kompute Algorithm initialising with tensor size: {} and "
"spirv size: {}",
tensors.size(),
spirv.size());
this->rebuild(
tensors, spirv, workgroup, specializationConstants, pushConstants);
} else {
KP_LOG_INFO("Kompute Algorithm constructor with empty tensors and or "
"spirv so not rebuilding vulkan components");
}
}
/**
* Rebuild function to reconstruct algorithm with configuration parameters
@ -1116,11 +1134,57 @@ class Algorithm
* these can be modified but all new values must have the same vector size
* as this initial value.
*/
template<typename S = float, typename P = float>
void rebuild(const std::vector<std::shared_ptr<Tensor>>& tensors,
const std::vector<uint32_t>& spirv,
const Workgroup& workgroup = {},
const Constants& specializationConstants = {},
const Constants& pushConstants = {});
const std::vector<S>& specializationConstants = {},
const std::vector<P>& pushConstants = {})
{
KP_LOG_DEBUG("Kompute Algorithm rebuild started");
this->mTensors = tensors;
this->mSpirv = spirv;
if (specializationConstants.size()) {
if (this->mSpecializationConstantsData) {
free(this->mSpecializationConstantsData);
}
uint32_t memorySize = sizeof(decltype(specializationConstants.back()));
uint32_t size = specializationConstants.size();
uint32_t totalSize = size * memorySize;
this->mSpecializationConstantsData = malloc(totalSize);
memcpy(this->mSpecializationConstantsData, specializationConstants.data(), totalSize);
this->mSpecializationConstantsDataTypeMemorySize = memorySize;
this->mSpecializationConstantsSize = size;
}
if (pushConstants.size()) {
if (this->mPushConstantsData) {
free(this->mPushConstantsData);
}
uint32_t memorySize = sizeof(decltype(pushConstants.back()));
uint32_t size = pushConstants.size();
uint32_t totalSize = size * memorySize;
this->mPushConstantsData = malloc(totalSize);
memcpy(this->mPushConstantsData, pushConstants.data(), totalSize);
this->mPushConstantsDataTypeMemorySize = memorySize;
this->mPushConstantsSize = size;
}
this->setWorkgroup(workgroup,
this->mTensors.size() ? this->mTensors[0]->size() : 1);
// Descriptor pool is created first so if available then destroy all before
// rebuild
if (this->isInit()) {
this->destroy();
}
this->createParameters();
this->createShaderModule();
this->createPipeline();
}
/**
* Destructor for Algorithm which is responsible for freeing and desroying
@ -1179,7 +1243,29 @@ class Algorithm
* next bindPush(...) calls. The constants provided must be of the same size
* as the ones created during initialization.
*/
void setPush(const Constants& pushConstants);
template<typename T>
void setPushConstants(const std::vector<T>& pushConstants)
{
if (pushConstants.size() != this->mPushConstantsSize) {
throw std::runtime_error(
fmt::format("Kompute Algorithm push "
"constant provided is size {} but expected size {}",
pushConstants.size(),
this->mPushConstantsSize));
}
if (this->mPushConstantsData) {
free(this->mPushConstantsData);
}
uint32_t memorySize = sizeof(decltype(pushConstants.back()));
uint32_t size = pushConstants.size();
uint32_t totalSize = size * memorySize;
this->mPushConstantsData = malloc(totalSize);
memcpy(this->mPushConstantsData, pushConstants.data(), totalSize);
this->mPushConstantsDataTypeMemorySize = memorySize;
this->mPushConstantsSize = size;
}
/**
* Gets the current workgroup from the algorithm.
@ -1194,13 +1280,23 @@ class Algorithm
*
* @returns The kp::Constants currently set for specialization constants
*/
const Constants& getSpecializationConstants();
template<typename T>
const std::vector<T> getSpecializationConstants()
{
return { (T*)this->mSpecializationConstantsData,
((T*)this->mSpecializationConstantsData) + this->mSpecializationConstantsSize };
}
/**
* Gets the specialization constants of the current algorithm.
*
* @returns The kp::Constants currently set for push constants
*/
const Constants& getPush();
template<typename T>
const std::vector<T> getPushConstants()
{
return { (T*)this->mPushConstantsData,
((T*)this->mPushConstantsData) + this->mPushConstantsSize };
}
/**
* Gets the current tensors that are used in the algorithm.
*
@ -1233,8 +1329,12 @@ class Algorithm
// -------------- ALWAYS OWNED RESOURCES
std::vector<uint32_t> mSpirv;
Constants mSpecializationConstants;
Constants mPushConstants;
void* mSpecializationConstantsData = nullptr;
uint32_t mSpecializationConstantsDataTypeMemorySize = 0;
uint32_t mSpecializationConstantsSize = 0;
void* mPushConstantsData = nullptr;
uint32_t mPushConstantsDataTypeMemorySize = 0;
uint32_t mPushConstantsSize = 0;
Workgroup mWorkgroup;
// Create util functions
@ -1655,7 +1755,7 @@ class OpMult : public OpAlgoDispatch
(uint32_t*)(shader_data::shaders_glsl_opmult_comp_spv +
kp::shader_data::shaders_glsl_opmult_comp_spv_len));
algorithm->rebuild(tensors, spirv);
algorithm->rebuild<>(tensors, spirv);
}
/**

114
src/Algorithm.cpp
View file

@ -5,30 +5,6 @@
namespace kp {
Algorithm::Algorithm(std::shared_ptr<vk::Device> device,
const std::vector<std::shared_ptr<Tensor>>& tensors,
const std::vector<uint32_t>& spirv,
const Workgroup& workgroup,
const Constants& specializationConstants,
const Constants& pushConstants)
{
KP_LOG_DEBUG("Kompute Algorithm Constructor with device");
this->mDevice = device;
if (tensors.size() && spirv.size()) {
KP_LOG_INFO("Kompute Algorithm initialising with tensor size: {} and "
"spirv size: {}",
tensors.size(),
spirv.size());
this->rebuild(
tensors, spirv, workgroup, specializationConstants, pushConstants);
} else {
KP_LOG_INFO("Kompute Algorithm constructor with empty tensors and or "
"spirv so not rebuilding vulkan components");
}
}
Algorithm::~Algorithm()
{
KP_LOG_DEBUG("Kompute Algorithm Destructor started");
@ -36,33 +12,6 @@ Algorithm::~Algorithm()
this->destroy();
}
void
Algorithm::rebuild(const std::vector<std::shared_ptr<Tensor>>& tensors,
const std::vector<uint32_t>& spirv,
const Workgroup& workgroup,
const Constants& specializationConstants,
const Constants& pushConstants)
{
KP_LOG_DEBUG("Kompute Algorithm rebuild started");
this->mTensors = tensors;
this->mSpirv = spirv;
this->mSpecializationConstants = specializationConstants;
this->mPushConstants = pushConstants;
this->setWorkgroup(workgroup,
this->mTensors.size() ? this->mTensors[0]->size() : 1);
// Descriptor pool is created first so if available then destroy all before
// rebuild
if (this->isInit()) {
this->destroy();
}
this->createParameters();
this->createShaderModule();
this->createPipeline();
}
bool
Algorithm::isInit()
{
@ -74,6 +23,13 @@ Algorithm::isInit()
void
Algorithm::destroy()
{
if (this->mPushConstantsData) {
free(this->mPushConstantsData);
}
if (this->mSpecializationConstantsData) {
free(this->mSpecializationConstantsData);
}
if (!this->mDevice) {
KP_LOG_WARN("Kompute Algorithm destroy function reached with null "
@ -279,10 +235,10 @@ Algorithm::createPipeline()
this->mDescriptorSetLayout.get());
vk::PushConstantRange pushConstantRange;
if (this->mPushConstants.size()) {
if (this->mPushConstantsSize) {
pushConstantRange.setStageFlags(vk::ShaderStageFlagBits::eCompute);
pushConstantRange.setOffset(0);
pushConstantRange.setSize(sizeof(float) * this->mPushConstants.size());
pushConstantRange.setSize(this->mPushConstantsDataTypeMemorySize * this->mPushConstantsSize);
pipelineLayoutInfo.setPushConstantRangeCount(1);
pipelineLayoutInfo.setPPushConstantRanges(&pushConstantRange);
@ -295,11 +251,11 @@ Algorithm::createPipeline()
std::vector<vk::SpecializationMapEntry> specializationEntries;
for (uint32_t i = 0; i < this->mSpecializationConstants.size(); i++) {
for (uint32_t i = 0; i < this->mSpecializationConstantsSize; i++) {
vk::SpecializationMapEntry specializationEntry(
static_cast<uint32_t>(i),
static_cast<uint32_t>(sizeof(float) * i),
sizeof(float));
static_cast<uint32_t>(this->mSpecializationConstantsDataTypeMemorySize * i),
this->mSpecializationConstantsDataTypeMemorySize);
specializationEntries.push_back(specializationEntry);
}
@ -309,8 +265,8 @@ Algorithm::createPipeline()
vk::SpecializationInfo specializationInfo(
static_cast<uint32_t>(specializationEntries.size()),
specializationEntries.data(),
sizeof(float) * this->mSpecializationConstants.size(),
this->mSpecializationConstants.data());
this->mSpecializationConstantsDataTypeMemorySize * this->mSpecializationConstantsSize,
this->mSpecializationConstantsData);
vk::PipelineShaderStageCreateInfo shaderStage(
vk::PipelineShaderStageCreateFlags(),
@ -381,15 +337,22 @@ Algorithm::recordBindCore(const vk::CommandBuffer& commandBuffer)
void
Algorithm::recordBindPush(const vk::CommandBuffer& commandBuffer)
{
if (this->mPushConstants.size()) {
if (this->mPushConstantsSize) {
KP_LOG_DEBUG("Kompute Algorithm binding push constants size: {}",
this->mPushConstants.size());
this->mPushConstantsSize);
KP_LOG_DEBUG("{} {}",
this->mPushConstantsDataTypeMemorySize,
this->mPushConstantsData == nullptr);
KP_LOG_DEBUG("{}",
((float*)this->mPushConstantsData)[0]);
commandBuffer.pushConstants(*this->mPipelineLayout,
vk::ShaderStageFlagBits::eCompute,
0,
this->mPushConstants.size() * sizeof(float),
this->mPushConstants.data());
this->mPushConstantsSize * this->mPushConstantsDataTypeMemorySize,
this->mPushConstantsData);
KP_LOG_DEBUG("Constants bound: {}",
this->mPushConstantsSize);
}
}
@ -426,39 +389,12 @@ Algorithm::setWorkgroup(const Workgroup& workgroup, uint32_t minSize)
this->mWorkgroup[2]);
}
void
Algorithm::setPush(const Constants& pushConstants)
{
if (pushConstants.size() != this->mPushConstants.size()) {
throw std::runtime_error(
fmt::format("Kompute Algorithm push "
"constant provided is size {} but expected size {}",
pushConstants.size(),
this->mPushConstants.size()));
}
this->mPushConstants = pushConstants;
}
const Workgroup&
Algorithm::getWorkgroup()
{
return this->mWorkgroup;
}
const Constants&
Algorithm::getSpecializationConstants()
{
return this->mSpecializationConstants;
}
const Constants&
Algorithm::getPush()
{
return this->mPushConstants;
}
const std::vector<std::shared_ptr<Tensor>>&
Algorithm::getTensors()
{

1
src/Manager.cpp
View file

@ -422,6 +422,7 @@ Manager::createDevice(const std::vector<uint32_t>& familyQueueIndices,
KP_LOG_DEBUG("Kompute Manager compute queue obtained");
}
// TODO: Update to template
std::shared_ptr<Algorithm>
Manager::algorithm(const std::vector<std::shared_ptr<Tensor>>& tensors,
const std::vector<uint32_t>& spirv,

2
src/OpAlgoDispatch.cpp
View file

@ -36,7 +36,7 @@ OpAlgoDispatch::record(const vk::CommandBuffer& commandBuffer)
}
if (this->mPushConstants.size()) {
this->mAlgorithm->setPush(this->mPushConstants);
this->mAlgorithm->setPushConstants(this->mPushConstants);
}
this->mAlgorithm->recordBindCore(commandBuffer);

118
src/include/kompute/Algorithm.hpp
View file

@ -31,12 +31,30 @@ class Algorithm
* these can be modified but all new values must have the same vector size
* as this initial value.
*/
template<typename S = float, typename P = float>
Algorithm(std::shared_ptr<vk::Device> device,
const std::vector<std::shared_ptr<Tensor>>& tensors = {},
const std::vector<uint32_t>& spirv = {},
const Workgroup& workgroup = {},
const Constants& specializationConstants = {},
const Constants& pushConstants = {});
const std::vector<S>& specializationConstants = {},
const std::vector<P>& pushConstants = {})
{
KP_LOG_DEBUG("Kompute Algorithm Constructor with device");
this->mDevice = device;
if (tensors.size() && spirv.size()) {
KP_LOG_INFO("Kompute Algorithm initialising with tensor size: {} and "
"spirv size: {}",
tensors.size(),
spirv.size());
this->rebuild(
tensors, spirv, workgroup, specializationConstants, pushConstants);
} else {
KP_LOG_INFO("Kompute Algorithm constructor with empty tensors and or "
"spirv so not rebuilding vulkan components");
}
}
/**
* Rebuild function to reconstruct algorithm with configuration parameters
@ -53,11 +71,57 @@ class Algorithm
* these can be modified but all new values must have the same vector size
* as this initial value.
*/
template<typename S = float, typename P = float>
void rebuild(const std::vector<std::shared_ptr<Tensor>>& tensors,
const std::vector<uint32_t>& spirv,
const Workgroup& workgroup = {},
const Constants& specializationConstants = {},
const Constants& pushConstants = {});
const std::vector<S>& specializationConstants = {},
const std::vector<P>& pushConstants = {})
{
KP_LOG_DEBUG("Kompute Algorithm rebuild started");
this->mTensors = tensors;
this->mSpirv = spirv;
if (specializationConstants.size()) {
if (this->mSpecializationConstantsData) {
free(this->mSpecializationConstantsData);
}
uint32_t memorySize = sizeof(decltype(specializationConstants.back()));
uint32_t size = specializationConstants.size();
uint32_t totalSize = size * memorySize;
this->mSpecializationConstantsData = malloc(totalSize);
memcpy(this->mSpecializationConstantsData, specializationConstants.data(), totalSize);
this->mSpecializationConstantsDataTypeMemorySize = memorySize;
this->mSpecializationConstantsSize = size;
}
if (pushConstants.size()) {
if (this->mPushConstantsData) {
free(this->mPushConstantsData);
}
uint32_t memorySize = sizeof(decltype(pushConstants.back()));
uint32_t size = pushConstants.size();
uint32_t totalSize = size * memorySize;
this->mPushConstantsData = malloc(totalSize);
memcpy(this->mPushConstantsData, pushConstants.data(), totalSize);
this->mPushConstantsDataTypeMemorySize = memorySize;
this->mPushConstantsSize = size;
}
this->setWorkgroup(workgroup,
this->mTensors.size() ? this->mTensors[0]->size() : 1);
// Descriptor pool is created first so if available then destroy all before
// rebuild
if (this->isInit()) {
this->destroy();
}
this->createParameters();
this->createShaderModule();
this->createPipeline();
}
/**
* Destructor for Algorithm which is responsible for freeing and desroying
@ -116,7 +180,29 @@ class Algorithm
* next bindPush(...) calls. The constants provided must be of the same size
* as the ones created during initialization.
*/
void setPush(const Constants& pushConstants);
template<typename T>
void setPushConstants(const std::vector<T>& pushConstants)
{
if (pushConstants.size() != this->mPushConstantsSize) {
throw std::runtime_error(
fmt::format("Kompute Algorithm push "
"constant provided is size {} but expected size {}",
pushConstants.size(),
this->mPushConstantsSize));
}
if (this->mPushConstantsData) {
free(this->mPushConstantsData);
}
uint32_t memorySize = sizeof(decltype(pushConstants.back()));
uint32_t size = pushConstants.size();
uint32_t totalSize = size * memorySize;
this->mPushConstantsData = malloc(totalSize);
memcpy(this->mPushConstantsData, pushConstants.data(), totalSize);
this->mPushConstantsDataTypeMemorySize = memorySize;
this->mPushConstantsSize = size;
}
/**
* Gets the current workgroup from the algorithm.
@ -131,13 +217,23 @@ class Algorithm
*
* @returns The kp::Constants currently set for specialization constants
*/
const Constants& getSpecializationConstants();
template<typename T>
const std::vector<T> getSpecializationConstants()
{
return { (T*)this->mSpecializationConstantsData,
((T*)this->mSpecializationConstantsData) + this->mSpecializationConstantsSize };
}
/**
* Gets the specialization constants of the current algorithm.
*
* @returns The kp::Constants currently set for push constants
*/
const Constants& getPush();
template<typename T>
const std::vector<T> getPushConstants()
{
return { (T*)this->mPushConstantsData,
((T*)this->mPushConstantsData) + this->mPushConstantsSize };
}
/**
* Gets the current tensors that are used in the algorithm.
*
@ -170,8 +266,12 @@ class Algorithm
// -------------- ALWAYS OWNED RESOURCES
std::vector<uint32_t> mSpirv;
Constants mSpecializationConstants;
Constants mPushConstants;
void* mSpecializationConstantsData = nullptr;
uint32_t mSpecializationConstantsDataTypeMemorySize = 0;
uint32_t mSpecializationConstantsSize = 0;
void* mPushConstantsData = nullptr;
uint32_t mPushConstantsDataTypeMemorySize = 0;
uint32_t mPushConstantsSize = 0;
Workgroup mWorkgroup;
// Create util functions

2
src/include/kompute/operations/OpMult.hpp
View file

@ -45,7 +45,7 @@ class OpMult : public OpAlgoDispatch
(uint32_t*)(shader_data::shaders_glsl_opmult_comp_spv +
kp::shader_data::shaders_glsl_opmult_comp_spv_len));
algorithm->rebuild(tensors, spirv);
algorithm->rebuild<>(tensors, spirv);
}
/**