thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Updated to function for async

Browse source
This commit is contained in:
Alejandro Saucedo 2020-10-15 09:25:16 +01:00
parent 33df1dec4e
commit 48805e1639
5 changed files with 138 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

33
src/include/kompute/Manager.hpp
View file

@ -102,7 +102,7 @@ class Manager
*
* @param tensors The tensors to be used in the operation recorded
* @param sequenceName The name of the sequence to be retrieved or created
* @param TArgs Template parameters that will be used to initialise
* @param params Template parameters that will be used to initialise
* Operation to allow for extensible configurations on initialisation
*/
template<typename T, typename... TArgs>
@ -110,49 +110,52 @@ class Manager
std::string sequenceName,
TArgs&&... params)
{
SPDLOG_DEBUG("Kompute Manager evalOp triggered");
SPDLOG_DEBUG("Kompute Manager evalOpAsync triggered");
std::weak_ptr<Sequence> sqWeakPtr =
this->getOrCreateManagedSequence(sequenceName);
if (std::shared_ptr<kp::Sequence> sq = sqWeakPtr.lock()) {
SPDLOG_DEBUG("Kompute Manager evalOp running sequence BEGIN");
SPDLOG_DEBUG("Kompute Manager evalOpAsync running sequence BEGIN");
sq->begin();
SPDLOG_DEBUG("Kompute Manager evalOp running sequence RECORD");
SPDLOG_DEBUG("Kompute Manager evalOpAsync running sequence RECORD");
sq->record<T>(tensors, std::forward<TArgs>(params)...);
SPDLOG_DEBUG("Kompute Manager evalOp running sequence END");
SPDLOG_DEBUG("Kompute Manager evalOpAsync running sequence END");
sq->end();
SPDLOG_DEBUG("Kompute Manager evalOp running sequence EVAL");
SPDLOG_DEBUG("Kompute Manager evalOpAsync running sequence EVAL");
sq->evalAsync();
}
SPDLOG_DEBUG("Kompute Manager evalOp running sequence SUCCESS");
SPDLOG_DEBUG("Kompute Manager evalOpAsync running sequence SUCCESS");
}
/**
* Operation that adds extra operations to existing or new created
* sequences.
*
* @param tensors The tensors to be used in the operation recorded
* @param sequenceName The name of the sequence to be retrieved or created
* @param TArgs Template parameters that will be used to initialise
* Operation to allow for extensible configurations on initialisation
* @param sequenceName The name of the sequence to wait for termination
* @param waitFor The amount of time to wait before timing out
*/
template<typename T, typename... TArgs>
template<typename... TArgs>
void evalOpAwait(std::string sequenceName, uint64_t waitFor = UINT64_MAX)
{
SPDLOG_DEBUG("Kompute Manager evalOpAwait triggered");
std::unordered_map<std::string, std::shared_ptr<Sequence>>::iterator found =
this->mManagedSequences.find(sequenceName);
this->mManagedSequences.find(sequenceName);
if (found == this->mManagedSequences.end()) {
if (found != this->mManagedSequences.end()) {
if (std::shared_ptr<kp::Sequence> sq = found->second) {
SPDLOG_DEBUG("Kompute Manager evalOpAwait running sequence Sequence EVAL AWAIT");
sq->evalAwait(waitFor);
if (sq->isRunning()) {
sq->evalAwait(waitFor);
}
}
SPDLOG_DEBUG("Kompute Manager evalOpAwait running sequence SUCCESS");
}
else {
SPDLOG_ERROR("Sequence not found");
}
}
/**

9
src/include/kompute/Sequence.hpp
View file

@ -88,6 +88,13 @@ class Sequence
*/
bool isRecording();
/**
* Returns true if the sequence is currently running - mostly used for async workloads.
*
* @return Boolean stating if currently running.
*/
bool isRunning();
/**
* Returns true if the sequence has been successfully initialised.
*
@ -164,7 +171,7 @@ class Sequence
// State
bool mIsInit = false;
bool mRecording = false;
bool mEvalBusy = false;
bool mIsRunning = false;
// Create functions
void createCommandPool();

2
src/include/kompute/operations/OpAlgoBase.hpp
View file

@ -162,7 +162,7 @@ OpAlgoBase<tX, tY, tZ>::OpAlgoBase(std::shared_ptr<vk::PhysicalDevice> physicalD
std::vector<std::shared_ptr<Tensor>>& tensors)
: OpBase(physicalDevice, device, commandBuffer, tensors, false)
{
SPDLOG_DEBUG("Kompute OpAlgoBase constructor with params numTensors: {} , shaderFilePath: {}", tensors.size());
SPDLOG_DEBUG("Kompute OpAlgoBase constructor with params numTensors: {}", tensors.size());
// The dispatch size is set up based on either explicitly provided template
// parameters or by default it would take the shape and size of the tensors