thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Added initial base for opcreatetensor

Browse source
This commit is contained in:
Alejandro Saucedo 2020-08-18 21:13:33 +01:00
parent 93041b4519
commit 014f15d552
14 changed files with 422 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.ccls
View file

@ -5,7 +5,7 @@
-fms-extensions
-Wall
-Wextra
-std=c++11
-std=c++17
%h -x
%h c++-header

5
Makefile
View file

@ -14,9 +14,10 @@ SCMP=/c/VulkanSDK/1.2.141.2/Bin32/glslangValidator.exe
build: build_shaders
$(CC) \
-g -fexceptions -fPIC \
src/* \
-std=c++14 \
-std=c++17 \
-g -fexceptions -fPIC \
-static-libgcc -static-libstdc++ \
-DDEBUG=1 \
-I"./external/" \
-I"./src/" \

32
README.md
View file

@ -11,6 +11,21 @@
Use default equations
```c++
int main() {
kp::Manager kManager(); // Chooses device 0
kp::Tensor inputOne = kp::Tensor({0, 1, 2, 3});
kp::Tensor inputTwo;
inputTwo = kManager.eval<kp::OpCreateTensor>(&inputTwo);
kp::Tensor output = kManager.eval<kp::OpMult>(inputOne, inputTwo);
std::cout << output << std::endl;
}
```
```c++
int main() {
kp::Manager kManager(); // Chooses device 0
@ -18,6 +33,13 @@ int main() {
kp::Tensor inputOne = kManager.eval<kp::OpCreateTensor>({0, 1, 2, 3}); // Mounts to device and binds to 0
kp::Tensor inputTwo = kManager.eval<kp::OpCreateTensor>({0, 1, 2, 3}); // Mounts to device and binds to 1
kp::Tensor inputOne({0, 1, 2, 3});
kManager.eval<kp::OpCreateTensor>(&inputOne); // Mounts to device and binds to 0
kp::Tensor inputOne({0, 1, 2, 3});
kManager.eval<kp::OpCreateTensor>(&inputTwo); // Mounts to device and binds to 0
kp::Tensor output = kManager.eval<kp::OpMult>(inputOne, inputTwo);
std::cout << output << std::endl;
@ -42,11 +64,9 @@ class CustomOp : kp::BaseOperator {
int main() {
kp::Manager kManager(); // Chooses device 0
kp::Tensor inputOne;
kManager.eval<kp::OpCreateTensor>(&inputOne, {0, 1, 2, 3}); // Mounts to device and binds to 0
kp::Tensor inputOne({0, 1, 2, 3});
kp::Tensor inputTwo;
kManager.eval<kp::OpCreateTensor>(&inputTwo, {0, 1, 2, 3}); // Mounts to device and binds to 1
kp::Tensor inputTwo({0, 1, 2, 3});
kp::Tensor output;
kManager.eval<kp::CustomOp>(&inputOne, &inputTwo, &output);
@ -63,7 +83,8 @@ int main() {
kp::Sequence sq;
kManager.createSequence(&sq);
//kManager.createSequence<kp::Sequence>();
sq.begin();
kp::Tensor inputOne;
sq.record<kp::OpCreateTensor>(&inputOne, {0, 1, 2, 3}); // Mounts to device and binds to 0
@ -74,6 +95,7 @@ int main() {
kp::Tensor output;
sq.record<kp::OpMult>(&inputOne, &inputTwo, &output);
sq.end();
sq.eval();
std::cout << output << std::endl;

27
src/BaseOp.cpp
View file

@ -10,29 +10,26 @@
namespace kp {
BaseOp::BaseOp() {}
template<class T>
BaseOp<T>::BaseOp()
{}
BaseOp::BaseOp(std::shared_ptr<vk::CommandBuffer> commandBuffer)
template<class T>
BaseOp<T>::BaseOp(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer)
{
SPDLOG_DEBUG("Compute BaseOp constructor started");
this->mPhysicalDevice = physicalDevice;
this->mDevice = device;
this->mCommandBuffer = commandBuffer;
}
BaseOp::~BaseOp()
template<class T>
BaseOp<T>::~BaseOp()
{
SPDLOG_DEBUG("Compute BaseOp destructor started");
}
void
BaseOp::init(std::string one, std::string two)
{
SPDLOG_DEBUG("Compute BaseOp init started");
}
void
BaseOp::record()
{
SPDLOG_DEBUG("Compute BaseOp record started");
}
}

30
src/BaseOp.hpp
View file

@ -5,20 +5,42 @@
#include <vulkan/vulkan.h>
#include <vulkan/vulkan.hpp>
// SPDLOG_ACTIVE_LEVEL must be defined before spdlog.h import
#if DEBUG
#define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_DEBUG
#endif
#include <spdlog/spdlog.h>
namespace kp {
template<class T>
class BaseOp
{
private:
public:
BaseOp();
BaseOp(std::shared_ptr<vk::CommandBuffer> commandBuffer);
BaseOp(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer);
virtual ~BaseOp();
void init(std::string one, std::string two);
void record();
template<typename... TArgs>
void init(TArgs&&... args)
{
SPDLOG_DEBUG("Compute BaseOp init started");
static_cast<T*>(this)->init(std::forward<TArgs>(args)...);
}
private:
template<typename... TArgs>
void record(TArgs&&... args)
{
SPDLOG_DEBUG("Compute BaseOp record started");
static_cast<T*>(this)->record(std::forward<TArgs>(args)...);
}
protected:
std::shared_ptr<vk::PhysicalDevice> mPhysicalDevice;
std::shared_ptr<vk::Device> mDevice;
std::shared_ptr<vk::CommandBuffer> mCommandBuffer;
};

3
src/Manager.cpp
View file

@ -166,6 +166,9 @@ Manager::createDevice()
vk::PhysicalDevice physicalDevice =
physicalDevices[this->mPhysicalDeviceIndex];
this->mPhysicalDevice =
std::make_shared<vk::PhysicalDevice>(physicalDevice);
vk::PhysicalDeviceProperties physicalDeviceProperties =
physicalDevice.getProperties();

10
src/Manager.hpp
View file

@ -20,18 +20,19 @@ class Manager
public:
Manager();
// Manager(std::shared_ptr<vk::Instance> instance,
// std::shared_ptr<vk::Device>, uint32_t queueIndex);
Manager(std::shared_ptr<vk::Instance> instance,
std::shared_ptr<vk::Device>,
uint32_t queueIndex);
~Manager();
// Evaluate actions
template<typename T, typename... TArgs>
void eval(TArgs&&... args)
void evalOp(TArgs&&... args)
{
SPDLOG_DEBUG("Kompute Manager eval triggered");
Sequence sq(
this->mDevice, this->mComputeQueue, this->mComputeQueueFamilyIndex);
this->mPhysicalDevice, this->mDevice, this->mComputeQueue, this->mComputeQueueFamilyIndex);
SPDLOG_DEBUG("Kompute Manager created sequence");
sq.begin();
SPDLOG_DEBUG("Kompute Manager sequence begin");
@ -46,6 +47,7 @@ class Manager
private:
std::shared_ptr<vk::Instance> mInstance = nullptr;
bool mFreeInstance = false;
std::shared_ptr<vk::PhysicalDevice> mPhysicalDevice = nullptr;
uint32_t mPhysicalDeviceIndex = -1;
std::shared_ptr<vk::Device> mDevice = nullptr;
bool mFreeDevice = false;

43
src/OpCreateTensor.cpp
View file

@ -1,15 +1,48 @@
#include "Tensor.hpp"
#include "OpCreateTensor.hpp"
namespace kp {
OpCreateTensor::OpCreateTensor() {}
OpCreateTensor::OpCreateTensor(std::shared_ptr<vk::CommandBuffer> commandBuffer)
OpCreateTensor::OpCreateTensor(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer)
: BaseOp(physicalDevice, device, commandBuffer)
{
this->mCommandBuffer = commandBuffer;
}
OpCreateTensor::~OpCreateTensor() {}
}
OpCreateTensor::~OpCreateTensor() {
}
void
OpCreateTensor::init(std::shared_ptr<Tensor> tensor, std::vector<uint32_t> data)
{
this->mPrimaryTensor = tensor;
if (tensor->tensorType() == Tensor::TensorTypes::eDevice) {
tensor->init(this->mPhysicalDevice, this->mDevice, this->mCommandBuffer);
this->mStagingTensor = std::make_unique<Tensor>(tensor->shape(), Tensor::TensorTypes::eStaging);
this->mStagingTensor->init(this->mPhysicalDevice, this->mDevice, this->mCommandBuffer, data);
}
else {
tensor->init(this->mPhysicalDevice, this->mDevice, this->mCommandBuffer, data);
}
}
void
OpCreateTensor::record()
{
if (this->mPrimaryTensor->tensorType() == Tensor::TensorTypes::eDevice) {
this->mPrimaryTensor->recordCopyFrom(this->mStagingTensor);
}
}
}

17
src/OpCreateTensor.hpp
View file

@ -10,22 +10,31 @@
#include <spdlog/spdlog.h>
#include "Tensor.hpp"
#include "BaseOp.hpp"
namespace kp {
class OpCreateTensor : BaseOp
class OpCreateTensor : BaseOp<OpCreateTensor>
{
private:
public:
OpCreateTensor();
OpCreateTensor(std::shared_ptr<vk::CommandBuffer> commandBuffer);
OpCreateTensor(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::CommandBuffer> commandBuffer);
~OpCreateTensor();
void init(std::shared_ptr<Tensor> tensor, std::vector<uint32_t> data);
void record();
private:
std::shared_ptr<vk::CommandBuffer> mCommandBuffer;
std::shared_ptr<Tensor> mPrimaryTensor;
std::shared_ptr<Tensor> mStagingTensor;
};
} // End namespace kp

29
src/Sequence.cpp
View file

@ -8,12 +8,14 @@ Sequence::Sequence()
SPDLOG_DEBUG("Kompute Sequence base constructor");
}
Sequence::Sequence(std::shared_ptr<vk::Device> device,
Sequence::Sequence(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::Queue> computeQueue,
uint32_t queueIndex)
{
SPDLOG_DEBUG("Kompute Sequence Constructor with existing device & queue");
this->mPhysicalDevice = physicalDevice;
this->mDevice = device;
this->mComputeQueue = computeQueue;
this->mQueueIndex = queueIndex;
@ -26,33 +28,32 @@ Sequence::~Sequence()
{
SPDLOG_DEBUG("Kompute Sequence Destructor started");
if (this->mDevice == nullptr) {
if (!this->mDevice) {
spdlog::error(
"Kompute Sequence destructor reached with null Device pointer");
return;
}
if (this->mCommandBuffer == nullptr) {
spdlog::error(
"Kompute Sequence destructor reached with null CommandPool pointer");
return;
}
if (this->mFreeCommandBuffer) {
spdlog::info("Freeing CommandBuffer");
if (!this->mCommandBuffer) {
spdlog::error(
"Kompute Sequence destructor reached with null CommandPool pointer");
return;
}
this->mDevice->freeCommandBuffers(
*this->mCommandPool, 1, this->mCommandBuffer.get());
SPDLOG_DEBUG("Kompute Manager Freed CommandBuffer");
}
if (this->mCommandPool == nullptr) {
spdlog::error(
"Kompute Sequence destructor reached with null CommandPool pointer");
return;
}
if (this->mFreeCommandPool) {
spdlog::info("Destroying CommandPool");
if (this->mCommandPool == nullptr) {
spdlog::error(
"Kompute Sequence destructor reached with null CommandPool pointer");
return;
}
this->mDevice->destroy(*this->mCommandPool);
SPDLOG_DEBUG("Kompute Manager Destroyed CommandPool");
}
@ -120,11 +121,9 @@ Sequence::createCommandPool()
{
SPDLOG_DEBUG("Kompute Sequence creating command pool");
if (this->mDevice == nullptr) {
spdlog::info("cmdpoolinfo");
throw std::runtime_error("Kompute Sequence device is null");
}
if (this->mQueueIndex < 0) {
spdlog::info("Queue index {}", this->mQueueIndex);
throw std::runtime_error("Kompute Sequence queue index not provided");
}

4
src/Sequence.hpp
View file

@ -17,7 +17,8 @@ class Sequence
private:
public:
Sequence();
Sequence(std::shared_ptr<vk::Device> device,
Sequence(std::shared_ptr<vk::PhysicalDevice> physicalDevice,
std::shared_ptr<vk::Device> device,
std::shared_ptr<vk::Queue> computeQueue,
uint32_t queueIndex);
~Sequence();
@ -37,6 +38,7 @@ class Sequence
}
private:
std::shared_ptr<vk::PhysicalDevice> mPhysicalDevice = nullptr;
std::shared_ptr<vk::Device> mDevice = nullptr;
std::shared_ptr<vk::Queue> mComputeQueue = nullptr;
uint32_t mQueueIndex = -1;

204
src/Tensor.cpp Normal file
View file

@ -0,0 +1,204 @@
#include "Tensor.hpp"
namespace kp {
Tensor::Tensor() {
this->mTensorType = TensorTypes::eDevice;
}
Tensor::Tensor(std::vector<uint32_t> shape, TensorTypes tensorType)
{
SPDLOG_DEBUG("Kompute Tensor init with data");
if (shape.size() > KP_MAX_DIM_SIZE) {
spdlog::warn("Kompute Tensor created with more dimensions than supported. Max: {}, Provided: {}.", KP_MAX_DIM_SIZE, shape.size());
}
std::copy_n(shape.begin(), KP_MAX_DIM_SIZE, this->mShape.begin());
this->mTensorType = tensorType;
}
Tensor::~Tensor()
{
SPDLOG_DEBUG("Kompute Tensor destructor started");
if (!this->mDevice) {
spdlog::error(
"Kompute Sequence destructor reached with null Device pointer");
return;
}
if (this->mFreeBuffer) {
if (!this->mBuffer) {
spdlog::error("Kompose Tensor expected to free buffer but got null buffer");
} else {
this->mDevice->destroy(*this->mBuffer);
}
}
if (this->mFreeMemory) {
if (!this->mMemory) {
spdlog::error("Kompose Tensor expected to free buffer but got null memory");
} else {
this->mDevice->freeMemory(*this->mMemory);
}
}
}
void Tensor::init(std::shared_ptr<vk::PhysicalDevice> physicalDevice, std::shared_ptr<vk::Device> device, std::shared_ptr<vk::CommandBuffer> commandBuffer, std::vector<uint32_t> data) {
SPDLOG_DEBUG("Kompute Tensor running init with physicalDevice and logical device");
this->mPhysicalDevice = physicalDevice;
this->mDevice = device;
this->mCommandBuffer = commandBuffer;
this->mIsInit = true;
this->createBuffer(data.data());
}
std::vector<uint32_t> Tensor::data() {
return this->mData;
}
uint64_t Tensor::memorySize() {
return this->size() * sizeof(uint32_t);
}
uint32_t Tensor::size() {
return this->mShape[0];
}
std::array<uint32_t, KP_MAX_DIM_SIZE> Tensor::shape() {
return this->mShape;
}
Tensor::TensorTypes Tensor::tensorType() {
return this->mTensorType;
}
bool Tensor::isInit() {
return this->mIsInit;
}
void Tensor::recordCopyFrom(std::shared_ptr<Tensor> copyFromTensor) {
SPDLOG_DEBUG("Kompute Tensor recordCopyFrom called");
// TODO: Allow for dst and src offsets to be configured
// TODO: Test and ensure sizes for tensors are compatible
vk::DeviceSize bufferSize(this->memorySize());
vk::BufferCopy copyRegion(0, 0, bufferSize);
// TODO: Ensure command buffer is in same device from buffer
this->mCommandBuffer->copyBuffer(*copyFromTensor->mBuffer, *this->mBuffer, copyRegion);
this->mData = copyFromTensor->mData;
}
vk::BufferUsageFlags Tensor::getBufferUsageFlags() {
switch (this->mTensorType) {
case TensorTypes::eDevice:
return vk::BufferUsageFlagBits::eStorageBuffer |
vk::BufferUsageFlagBits::eTransferSrc |
vk::BufferUsageFlagBits::eTransferDst;
break;
case TensorTypes::eStaging:
return vk::BufferUsageFlagBits::eTransferSrc|
vk::BufferUsageFlagBits::eTransferDst;
break;
case TensorTypes::eStorage:
return vk::BufferUsageFlagBits::eStorageBuffer;
break;
default:
throw std::runtime_error("Invalid tensor type");
}
}
vk::MemoryPropertyFlags Tensor::getMemoryPropertyFlags() {
switch (this->mTensorType) {
case TensorTypes::eDevice:
return vk::MemoryPropertyFlagBits::eDeviceLocal;
break;
case TensorTypes::eStaging:
return vk::MemoryPropertyFlagBits::eHostVisible;
break;
case TensorTypes::eStorage:
return vk::MemoryPropertyFlagBits::eDeviceLocal;
break;
default:
throw std::runtime_error("Invalid tensor type");
}
}
void
Tensor::createBuffer(void* data)
{
SPDLOG_DEBUG("Kompute Tensor creating buffer");
if (!this->mIsInit) {
throw std::runtime_error("Kompute Tensor attempted to run createBuffer without init");
}
if (!this->mPhysicalDevice) {
throw std::runtime_error("Kompute Tensor phyisical device is null");
}
if (!this->mDevice) {
throw std::runtime_error("Kompute Tensor device is null");
}
this->mFreeBuffer = true;
vk::BufferUsageFlags usageFlags = this->getBufferUsageFlags();
vk::DeviceSize bufferSize = this->memorySize();
vk::BufferCreateInfo bufferInfo(vk::BufferCreateFlags(), bufferSize, usageFlags, vk::SharingMode::eExclusive);
this->mBuffer = std::make_shared<vk::Buffer>();
this->mDevice->createBuffer(&bufferInfo, nullptr, this->mBuffer.get());
vk::PhysicalDeviceMemoryProperties memoryProperties = this->mPhysicalDevice->getMemoryProperties();
vk::MemoryRequirements memoryRequirements = this->mDevice->getBufferMemoryRequirements(*this->mBuffer);
vk::MemoryPropertyFlags memoryPropertyFlags =
this->getMemoryPropertyFlags();
uint32_t memoryTypeIndex = -1;
for (uint32_t i = 0; i < memoryProperties.memoryTypeCount; i++) {
if (memoryRequirements.memoryTypeBits & (1 << i)) {
if ((memoryProperties.memoryTypes[i].propertyFlags & memoryPropertyFlags) == memoryPropertyFlags) {
memoryTypeIndex = i;
break;
}
}
}
if (memoryTypeIndex < 0) {
throw std::runtime_error("Memory type index for buffer creation not found");
}
this->mFreeMemory = true;
vk::MemoryAllocateInfo memoryAllocateInfo(memoryRequirements.size, memoryTypeIndex);
this->mMemory = std::make_shared<vk::DeviceMemory>();
this->mDevice->allocateMemory(&memoryAllocateInfo, nullptr, this->mMemory.get());
this->mDevice->bindBufferMemory(*this->mBuffer, *this->mMemory, 0);
SPDLOG_DEBUG("Kompute Tensor buffer & memory creation successful");
if (data != nullptr) {
SPDLOG_DEBUG("Kompute Tensor copying data to buffer");
// TODO: Verify if flushed memory ranges should happend in sequence
void* mapped = this->mDevice->mapMemory(*this->mMemory, 0, bufferSize, vk::MemoryMapFlags());
memcpy(mapped, data, bufferSize);
vk::MappedMemoryRange mappedRange(*this->mMemory, 0, bufferSize);
this->mDevice->flushMappedMemoryRanges(1, &mappedRange);
this->mDevice->unmapMemory(*this->mMemory);
SPDLOG_DEBUG("Kompute Tensor successful copy data to tensor");
}
}
}

65
src/Tensor.hpp
View file

@ -1,13 +1,74 @@
#pragma once
#include <vulkan/vulkan.h>
#include <vulkan/vulkan.hpp>
// SPDLOG_ACTIVE_LEVEL must be defined before spdlog.h import
#if DEBUG
#define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_DEBUG
#endif
#include <spdlog/spdlog.h>
#define KP_MAX_DIM_SIZE 1
namespace kp {
class Tensor
{
private:
public:
enum class TensorTypes
{
eDevice = 0,
eStaging = 1,
eStorage = 2,
};
Tensor();
virtual ~Tensor();
Tensor(std::vector<uint32_t> shape, TensorTypes tensorType = TensorTypes::eDevice);
~Tensor();
void init(std::shared_ptr<vk::PhysicalDevice> physicalDevice, std::shared_ptr<vk::Device> device, std::shared_ptr<vk::CommandBuffer> commandBuffer, std::vector<uint32_t> data = std::vector<uint32_t>());
// Create functions
void createBuffer(void* data = nullptr);
// Getter functions
std::vector<uint32_t> data();
uint32_t size();
std::array<uint32_t, KP_MAX_DIM_SIZE> shape();
TensorTypes tensorType();
bool isInit();
// Record functions
void recordCopyFrom(std::shared_ptr<Tensor> copyFromTensor);
private:
std::shared_ptr<vk::PhysicalDevice> mPhysicalDevice;
std::shared_ptr<vk::Device> mDevice;
std::shared_ptr<vk::CommandBuffer> mCommandBuffer;
std::shared_ptr<vk::Buffer> mBuffer;
bool mFreeBuffer;
std::shared_ptr<vk::DeviceMemory> mMemory;
bool mFreeMemory;
std::vector<uint32_t> mData;
TensorTypes mTensorType = TensorTypes::eDevice;
std::array<uint32_t, KP_MAX_DIM_SIZE> mShape; // TODO: Only 1D supported
bool mIsInit = false;
// uint32_t mDataType;
// Private util functions
vk::BufferUsageFlags getBufferUsageFlags();
vk::MemoryPropertyFlags getMemoryPropertyFlags();
uint64_t memorySize();
};
} // End namespace kp

12
src/main.cpp
View file

@ -20,7 +20,8 @@
#include <vulkan/vulkan.h>
#include <vulkan/vulkan.hpp>
#include "BaseOp.hpp"
#include "OpCreateTensor.hpp"
#include "Tensor.hpp"
#include "Manager.hpp"
#define BUFFER_ELEMENTS 32
@ -619,9 +620,14 @@ main()
// Run Kompute
spdlog::info("Creating manager");
kp::Manager mgr;
spdlog::info("Calling manager eval");
mgr.eval<kp::BaseOp>("one", "two");
std::vector<uint32_t> data = {0.0, 1.0, 2.0};
kp::Tensor tensor({data.size()});
spdlog::info("Calling manager eval w opcreatetensor");
mgr.evalOp<kp::OpCreateTensor>(&tensor, data);
spdlog::info("Called manager eval success");
std::vector<uint32_t> outData = tensor.data();
spdlog::info("Output data: {}", outData);
return 0;
} catch (const std::exception& exc) {
spdlog::error(exc.what());