thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Reformatted

Browse source
This commit is contained in:
Alejandro Saucedo 2020-10-17 11:19:25 +01:00
parent 872a0bc717
commit 9a64339e95
8 changed files with 95 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/Algorithm.cpp
View file

@ -266,8 +266,7 @@ Algorithm::createPipeline(std::vector<uint32_t> specializationData)
#ifdef KOMPUTE_CREATE_PIPELINE_RESULT_VALUE
vk::ResultValue<vk::Pipeline> pipelineResult =
this->mDevice->createComputePipeline(*this->mPipelineCache,
pipelineInfo);
this->mDevice->createComputePipeline(*this->mPipelineCache, pipelineInfo);
if (pipelineResult.result != vk::Result::eSuccess) {
throw std::runtime_error("Failed to create pipeline result: " +

39
src/Manager.cpp Executable file → Normal file
View file

@ -28,7 +28,8 @@ Manager::Manager()
: Manager(0)
{}
Manager::Manager(uint32_t physicalDeviceIndex, const std::vector<uint32_t> & familyQueueIndeces)
Manager::Manager(uint32_t physicalDeviceIndex,
const std::vector<uint32_t>& familyQueueIndeces)
{
this->mPhysicalDeviceIndex = physicalDeviceIndex;
@ -105,9 +106,13 @@ Manager::getOrCreateManagedSequence(std::string sequenceName)
}
std::weak_ptr<Sequence>
Manager::createManagedSequence(std::string sequenceName, uint32_t queueIndex) {
Manager::createManagedSequence(std::string sequenceName, uint32_t queueIndex)
{
SPDLOG_DEBUG("Kompute Manager createManagedSequence with sequenceName: {} and queueIndex: {}", sequenceName, queueIndex);
SPDLOG_DEBUG("Kompute Manager createManagedSequence with sequenceName: {} "
"and queueIndex: {}",
sequenceName,
queueIndex);
std::shared_ptr<Sequence> sq =
std::make_shared<Sequence>(this->mPhysicalDevice,
@ -205,7 +210,7 @@ Manager::createInstance()
}
void
Manager::createDevice(const std::vector<uint32_t> & familyQueueIndeces)
Manager::createDevice(const std::vector<uint32_t>& familyQueueIndeces)
{
SPDLOG_DEBUG("Kompute Manager creating Device");
@ -246,7 +251,8 @@ Manager::createDevice(const std::vector<uint32_t> & familyQueueIndeces)
vk::QueueFamilyProperties queueFamilyProperties =
allQueueFamilyProperties[i];
if (queueFamilyProperties.queueFlags & vk::QueueFlagBits::eCompute) {
if (queueFamilyProperties.queueFlags &
vk::QueueFlagBits::eCompute) {
computeQueueFamilyIndex = i;
break;
}
@ -257,8 +263,7 @@ Manager::createDevice(const std::vector<uint32_t> & familyQueueIndeces)
}
this->mComputeQueueFamilyIndeces.push_back(computeQueueFamilyIndex);
}
else {
} else {
this->mComputeQueueFamilyIndeces = familyQueueIndeces;
}
@ -277,28 +282,28 @@ Manager::createDevice(const std::vector<uint32_t> & familyQueueIndeces)
// Creating the respective device queue
vk::DeviceQueueCreateInfo deviceQueueCreateInfo(
vk::DeviceQueueCreateFlags(),
familyQueueInfo.first,
familyQueueInfo.second,
familyQueuePriorities[familyQueueInfo.first].data());
vk::DeviceQueueCreateFlags(),
familyQueueInfo.first,
familyQueueInfo.second,
familyQueuePriorities[familyQueueInfo.first].data());
deviceQueueCreateInfos.push_back(deviceQueueCreateInfo);
}
vk::DeviceCreateInfo deviceCreateInfo(vk::DeviceCreateFlags(),
deviceQueueCreateInfos.size(),
deviceQueueCreateInfos.data());
deviceQueueCreateInfos.size(),
deviceQueueCreateInfos.data());
this->mDevice = std::make_shared<vk::Device>();
physicalDevice.createDevice(
&deviceCreateInfo, nullptr, this->mDevice.get());
SPDLOG_DEBUG("Kompute Manager device created");
for (const uint32_t & familyQueueIndex : this->mComputeQueueFamilyIndeces)
{
for (const uint32_t& familyQueueIndex : this->mComputeQueueFamilyIndeces) {
std::shared_ptr<vk::Queue> currQueue = std::make_shared<vk::Queue>();
this->mDevice->getQueue(
familyQueueIndex, familyQueueIndexCount[familyQueueIndex], currQueue.get());
this->mDevice->getQueue(familyQueueIndex,
familyQueueIndexCount[familyQueueIndex],
currQueue.get());
familyQueueIndexCount[familyQueueIndex]++;

9
src/Sequence.cpp
View file

@ -141,7 +141,8 @@ Sequence::evalAsync()
return false;
}
if (this->mIsRunning) {
SPDLOG_WARN("Kompute Sequence evalAsync called when an eval async was called without successful wait");
SPDLOG_WARN("Kompute Sequence evalAsync called when an eval async was "
"called without successful wait");
return false;
}
@ -172,7 +173,8 @@ Sequence::evalAwait(uint64_t waitFor)
return false;
}
vk::Result result = this->mDevice->waitForFences(1, &this->mFence, VK_TRUE, waitFor);
vk::Result result =
this->mDevice->waitForFences(1, &this->mFence, VK_TRUE, waitFor);
this->mDevice->destroy(this->mFence);
if (result == vk::Result::eTimeout) {
@ -191,7 +193,8 @@ Sequence::evalAwait(uint64_t waitFor)
}
bool
Sequence::isRunning() {
Sequence::isRunning()
{
return this->mIsRunning;
}

17
src/include/kompute/Core.hpp
View file

@ -18,10 +18,11 @@ static const char* KOMPUTE_LOG_TAG = "KomputeLog";
#ifndef KOMPUTE_VK_API_MINOR_VERSION
#define KOMPUTE_VK_API_MINOR_VERSION 1
#endif // KOMPUTE_VK_API_MINOR_VERSION
#define KOMPUTE_VK_API_VERSION VK_MAKE_VERSION(KOMPUTE_VK_API_MAJOR_VERSION, KOMPUTE_VK_API_MINOR_VERSION, 0)
#define KOMPUTE_VK_API_VERSION \
VK_MAKE_VERSION( \
KOMPUTE_VK_API_MAJOR_VERSION, KOMPUTE_VK_API_MINOR_VERSION, 0)
#endif // KOMPUTE_VK_API_VERSION
// SPDLOG_ACTIVE_LEVEL must be defined before spdlog.h import
#if !defined(SPDLOG_ACTIVE_LEVEL)
#if DEBUG
@ -40,7 +41,8 @@ static const char* KOMPUTE_LOG_TAG = "KomputeLog";
#define SPDLOG_DEBUG(message, ...)
#else
#if defined(VK_USE_PLATFORM_ANDROID_KHR)
#define SPDLOG_DEBUG(message, ...) ((void)__android_log_print(ANDROID_LOG_DEBUG, KOMPUTE_LOG_TAG, message))
#define SPDLOG_DEBUG(message, ...) \
((void)__android_log_print(ANDROID_LOG_DEBUG, KOMPUTE_LOG_TAG, message))
#else
#define SPDLOG_DEBUG(message, ...) \
std::cout << "DEBUG: " << message << std::endl
@ -50,7 +52,8 @@ static const char* KOMPUTE_LOG_TAG = "KomputeLog";
#define SPDLOG_INFO(message, ...)
#else
#if defined(VK_USE_PLATFORM_ANDROID_KHR)
#define SPDLOG_INFO(message, ...) ((void)__android_log_print(ANDROID_LOG_INFO, KOMPUTE_LOG_TAG, message))
#define SPDLOG_INFO(message, ...) \
((void)__android_log_print(ANDROID_LOG_INFO, KOMPUTE_LOG_TAG, message))
#else
#define SPDLOG_INFO(message, ...) std::cout << "INFO: " << message << std::endl
#endif // VK_USE_PLATFORM_ANDROID_KHR
@ -59,7 +62,8 @@ static const char* KOMPUTE_LOG_TAG = "KomputeLog";
#define SPDLOG_WARN(message, ...)
#else
#if defined(VK_USE_PLATFORM_ANDROID_KHR)
#define SPDLOG_WARN(message, ...) ((void)__android_log_print(ANDROID_LOG_INFO, KOMPUTE_LOG_TAG, message))
#define SPDLOG_WARN(message, ...) \
((void)__android_log_print(ANDROID_LOG_INFO, KOMPUTE_LOG_TAG, message))
#else
#define SPDLOG_WARN(message, ...) \
std::cout << "WARNING: " << message << std::endl
@ -69,7 +73,8 @@ static const char* KOMPUTE_LOG_TAG = "KomputeLog";
#define SPDLOG_ERROR(message, ...)
#else
#if defined(VK_USE_PLATFORM_ANDROID_KHR)
#define SPDLOG_ERROR(message, ...) ((void)__android_log_print(ANDROID_LOG_INFO, KOMPUTE_LOG_TAG, message))
#define SPDLOG_ERROR(message, ...) \
((void)__android_log_print(ANDROID_LOG_INFO, KOMPUTE_LOG_TAG, message))
#else
#define SPDLOG_ERROR(message, ...) \
std::cout << "ERROR: " << message << std::endl

58
src/include/kompute/Manager.hpp
View file

@ -17,7 +17,7 @@ namespace kp {
*/
class Manager
{
public:
public:
/**
Base constructor and default used which creates the base resources
including choosing the device 0 by default.
@ -25,14 +25,16 @@ public:
Manager();
/**
* Similar to base constructor but allows the user to provide the device
* they would like to create the resources on.
*
* @param physicalDeviceIndex The index of the physical device to use
* @param familyQueueIndeces (Optional) List of queue indeces to add for explicit allocation
* @param totalQueues The total number of compute queues to create.
*/
Manager(uint32_t physicalDeviceIndex, const std::vector<uint32_t> & familyQueueIndeces = {});
* Similar to base constructor but allows the user to provide the device
* they would like to create the resources on.
*
* @param physicalDeviceIndex The index of the physical device to use
* @param familyQueueIndeces (Optional) List of queue indeces to add for
* explicit allocation
* @param totalQueues The total number of compute queues to create.
*/
Manager(uint32_t physicalDeviceIndex,
const std::vector<uint32_t>& familyQueueIndeces = {});
/**
* Manager constructor which allows your own vulkan application to integrate
@ -67,14 +69,15 @@ public:
std::string sequenceName);
/**
* Create a new managed Kompute sequence so it's available within the manager.
* Create a new managed Kompute sequence so it's available within the
* manager.
*
* @param sequenceName The name for the named sequence to be created
* @param queueIndex The queue to use from the available queues
* @return Weak pointer to the manager owned sequence resource
*/
std::weak_ptr<Sequence> createManagedSequence(
std::string sequenceName, uint32_t queueIndex = 0);
std::weak_ptr<Sequence> createManagedSequence(std::string sequenceName,
uint32_t queueIndex = 0);
/**
* Operation that adds extra operations to existing or new created
@ -121,15 +124,15 @@ public:
*/
template<typename T, typename... TArgs>
void evalOpAsync(std::vector<std::shared_ptr<Tensor>> tensors,
std::string sequenceName,
TArgs&&... params)
std::string sequenceName,
TArgs&&... params)
{
SPDLOG_DEBUG("Kompute Manager evalOpAsync triggered");
std::weak_ptr<Sequence> sqWeakPtr =
this->getOrCreateManagedSequence(sequenceName);
std::unordered_map<std::string, std::shared_ptr<Sequence>>::iterator found =
this->mManagedSequences.find(sequenceName);
std::unordered_map<std::string, std::shared_ptr<Sequence>>::iterator
found = this->mManagedSequences.find(sequenceName);
if (found != this->mManagedSequences.end()) {
std::shared_ptr<Sequence> sq = found->second;
@ -145,9 +148,9 @@ public:
SPDLOG_DEBUG("Kompute Manager evalOpAsync running sequence EVAL");
sq->evalAsync();
}
else {
SPDLOG_ERROR("Kompute Manager evalOpAsync sequence [{}] not found", sequenceName);
} else {
SPDLOG_ERROR("Kompute Manager evalOpAsync sequence [{}] not found",
sequenceName);
}
SPDLOG_DEBUG("Kompute Manager evalOpAsync running sequence SUCCESS");
}
@ -163,21 +166,22 @@ public:
void evalOpAwait(std::string sequenceName, uint64_t waitFor = UINT64_MAX)
{
SPDLOG_DEBUG("Kompute Manager evalOpAwait triggered");
std::unordered_map<std::string, std::shared_ptr<Sequence>>::iterator found =
this->mManagedSequences.find(sequenceName);
std::unordered_map<std::string, std::shared_ptr<Sequence>>::iterator
found = this->mManagedSequences.find(sequenceName);
if (found != this->mManagedSequences.end()) {
if (std::shared_ptr<kp::Sequence> sq = found->second) {
SPDLOG_DEBUG("Kompute Manager evalOpAwait running sequence Sequence EVAL AWAIT");
SPDLOG_DEBUG("Kompute Manager evalOpAwait running sequence "
"Sequence EVAL AWAIT");
if (sq->isRunning()) {
sq->evalAwait(waitFor);
}
}
SPDLOG_DEBUG("Kompute Manager evalOpAwait running sequence SUCCESS");
}
else {
SPDLOG_DEBUG(
"Kompute Manager evalOpAwait running sequence SUCCESS");
} else {
SPDLOG_ERROR("Kompute Manager evalOpAwait Sequence not found");
}
}
}
/**
@ -247,7 +251,7 @@ public:
// Create functions
void createInstance();
void createDevice(const std::vector<uint32_t> & familyQueueIndeces = {});
void createDevice(const std::vector<uint32_t>& familyQueueIndeces = {});
};
} // End namespace kp

10
src/include/kompute/Sequence.hpp
View file

@ -67,14 +67,17 @@ class Sequence
bool eval();
/**
* Eval Async sends all the recorded and stored operations in the vector of operations into the gpu as a submit job with a barrier. EvalAwait() must be called after to ensure the sequence is terminated correctly.
* Eval Async sends all the recorded and stored operations in the vector of
* operations into the gpu as a submit job with a barrier. EvalAwait() must
* be called after to ensure the sequence is terminated correctly.
*
* @return Boolean stating whether execution was successful.
*/
bool evalAsync();
/**
* Eval Await waits for the fence to finish processing and then once it finishes, it runs the postEval of all operations.
* Eval Await waits for the fence to finish processing and then once it
* finishes, it runs the postEval of all operations.
*
* @param waitFor Number of milliseconds to wait before timing out.
* @return Boolean stating whether execution was successful.
@ -89,7 +92,8 @@ class Sequence
bool isRecording();
/**
* Returns true if the sequence is currently running - mostly used for async workloads.
* Returns true if the sequence is currently running - mostly used for async
* workloads.
*
* @return Boolean stating if currently running.
*/

19
test/TestAsyncOperations.cpp Executable file → Normal file
View file

@ -72,13 +72,14 @@ TEST(TestAsyncOperations, TestManagerAsync)
for (uint32_t i = 0; i < numParallel; i++) {
mgr.evalOpDefault<kp::OpAlgoBase<>>(
{ inputsSyncA[i], inputsSyncB[i] },
{ inputsSyncA[i], inputsSyncB[i] },
std::vector<char>(shader.begin(), shader.end()));
}
auto endSync = std::chrono::high_resolution_clock::now();
auto durationSync = std::chrono::duration_cast<std::chrono::microseconds>(endSync - startSync).count();
auto durationSync =
std::chrono::duration_cast<std::chrono::microseconds>(endSync - startSync)
.count();
mgr.evalOpDefault<kp::OpTensorSyncLocal>(inputsSyncB);
@ -86,7 +87,7 @@ TEST(TestAsyncOperations, TestManagerAsync)
EXPECT_EQ(inputsSyncB[i]->data(), resultSync);
}
kp::Manager mgrAsync(0, {0, 2});
kp::Manager mgrAsync(0, { 0, 2 });
std::vector<std::shared_ptr<kp::Tensor>> inputsAsyncA;
std::vector<std::shared_ptr<kp::Tensor>> inputsAsyncB;
@ -107,8 +108,8 @@ TEST(TestAsyncOperations, TestManagerAsync)
for (uint32_t i = 0; i < numParallel; i++) {
mgrAsync.evalOpAsync<kp::OpAlgoBase<>>(
{ inputsAsyncA[i], inputsAsyncB[i] },
"async" + std::to_string(i),
{ inputsAsyncA[i], inputsAsyncB[i] },
"async" + std::to_string(i),
std::vector<char>(shader.begin(), shader.end()));
}
@ -117,7 +118,9 @@ TEST(TestAsyncOperations, TestManagerAsync)
}
auto endAsync = std::chrono::high_resolution_clock::now();
auto durationAsync = std::chrono::duration_cast<std::chrono::microseconds>(endAsync - startAsync).count();
auto durationAsync = std::chrono::duration_cast<std::chrono::microseconds>(
endAsync - startAsync)
.count();
mgrAsync.evalOpDefault<kp::OpTensorSyncLocal>({ inputsAsyncB });
@ -128,6 +131,6 @@ TEST(TestAsyncOperations, TestManagerAsync)
SPDLOG_ERROR("sync {}", durationSync);
SPDLOG_ERROR("async {}", durationAsync);
// The speedup should be at least 40%
// The speedup should be at least 40%
EXPECT_LT(durationAsync, durationSync * 0.6);
}

12
test/TestMultipleAlgoExecutions.cpp
View file

@ -258,10 +258,10 @@ TEST(TestMultipleAlgoExecutions, ManagerEvalMultSourceStrOpCreate)
)");
mgr.evalOpDefault<kp::OpAlgoBase<>>(
{ tensorInA, tensorInB, tensorOut },
std::vector<char>(shader.begin(), shader.end()));
{ tensorInA, tensorInB, tensorOut },
std::vector<char>(shader.begin(), shader.end()));
mgr.evalOpDefault<kp::OpTensorSyncLocal>({tensorOut});
mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorOut });
EXPECT_EQ(tensorOut->data(), std::vector<float>({ 0.0, 4.0, 12.0 }));
}
@ -295,10 +295,10 @@ TEST(TestMultipleAlgoExecutions, ManagerEvalMultSourceStrMgrCreate)
)");
mgr.evalOpDefault<kp::OpAlgoBase<>>(
{ tensorInA, tensorInB, tensorOut },
std::vector<char>(shader.begin(), shader.end()));
{ tensorInA, tensorInB, tensorOut },
std::vector<char>(shader.begin(), shader.end()));
mgr.evalOpDefault<kp::OpTensorSyncLocal>({tensorOut});
mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorOut });
EXPECT_EQ(tensorOut->data(), std::vector<float>({ 0.0, 4.0, 12.0 }));
}