thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Added rerecord functionality and tests

Browse source
This commit is contained in:
Alejandro Saucedo 2021-03-01 22:08:05 +00:00
parent 7f686b47da
commit 6192dda520
5 changed files with 110 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

61
single_include/kompute/Kompute.hpp
View file

@ -1146,8 +1146,8 @@ class Algorithm
* @specalizationInstalces The specialization parameters to pass to the
* function processing
*/
void rebuild(const std::vector<std::shared_ptr<Tensor>>& tensors = {},
const std::vector<uint32_t>& spirv = {},
void rebuild(const std::vector<std::shared_ptr<Tensor>>& tensors,
const std::vector<uint32_t>& spirv,
const Workgroup& workgroup = {},
const Constants& specializationConstants = {});
@ -1554,34 +1554,17 @@ class Sequence : public std::enable_shared_from_this<Sequence>
*/
template<typename T, typename... TArgs>
std::shared_ptr<Sequence> record(
std::vector<std::shared_ptr<Tensor>> tensors,
TArgs&&... params)
std::vector<std::shared_ptr<Tensor>> tensors, TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(tensors, std::forward<TArgs>(params)...) };
return this->record(op);
}
template<typename T, typename... TArgs>
std::shared_ptr<Sequence> record(std::shared_ptr<Algorithm> algorithm,
TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(algorithm,
std::forward<TArgs>(params)...) };
return this->record(op);
}
@ -1606,34 +1589,15 @@ class Sequence : public std::enable_shared_from_this<Sequence>
std::shared_ptr<Sequence> eval(std::vector<std::shared_ptr<Tensor>> tensors,
TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(tensors, std::forward<TArgs>(params)...) };
// TODO: Aim to be able to handle errors when returning without throw
// except
return this->eval(op);
}
// Needded as otherise can't use initialiser list
template<typename T, typename... TArgs>
std::shared_ptr<Sequence> eval(std::shared_ptr<Algorithm> algorithm,
TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(algorithm,
std::forward<TArgs>(params)...) };
return this->eval(op);
}
@ -1658,32 +1622,15 @@ class Sequence : public std::enable_shared_from_this<Sequence>
std::vector<std::shared_ptr<Tensor>> tensors,
TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(tensors, std::forward<TArgs>(params)...) };
return this->evalAsync(op);
}
// Needed as otherwise it's not possible to use initializer lists
template<typename T, typename... TArgs>
std::shared_ptr<Sequence> evalAsync(std::shared_ptr<Algorithm> algorithm,
TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(algorithm,
std::forward<TArgs>(params)...) };
return this->evalAsync(op);
}
@ -1727,6 +1674,8 @@ class Sequence : public std::enable_shared_from_this<Sequence>
bool isInit();
void rerecord();
/**
* Returns true if the sequence is currently running - mostly used for async
* workloads.

18
src/Sequence.cpp
View file

@ -51,6 +51,11 @@ Sequence::end()
{
KP_LOG_DEBUG("Kompute Sequence calling END");
if (this->isRunning()) {
throw std::runtime_error(
"Kompute Sequence begin called when sequence still running");
}
if (!this->isRecording()) {
KP_LOG_WARN("Kompute Sequence end called when not recording");
return;
@ -64,7 +69,7 @@ Sequence::end()
void
Sequence::clear()
{
KP_LOG_DEBUG("Kompute Sequence calling clear");
KP_LOG_DEBUG("Kompute Sequence calling clear");
this->end();
}
@ -171,6 +176,17 @@ Sequence::isInit()
this->mComputeQueue;
}
void
Sequence::rerecord()
{
this->end();
std::vector<std::shared_ptr<OpBase>> ops = this->mOperations;
this->mOperations.clear();
for (const std::shared_ptr<kp::OpBase>& op : ops) {
this->record(op);
}
}
void
Sequence::destroy()
{

4
src/include/kompute/Algorithm.hpp
View file

@ -35,8 +35,8 @@ class Algorithm
* @specalizationInstalces The specialization parameters to pass to the
* function processing
*/
void rebuild(const std::vector<std::shared_ptr<Tensor>>& tensors = {},
const std::vector<uint32_t>& spirv = {},
void rebuild(const std::vector<std::shared_ptr<Tensor>>& tensors,
const std::vector<uint32_t>& spirv,
const Workgroup& workgroup = {},
const Constants& specializationConstants = {});

57
src/include/kompute/Sequence.hpp
View file

@ -47,34 +47,17 @@ class Sequence : public std::enable_shared_from_this<Sequence>
*/
template<typename T, typename... TArgs>
std::shared_ptr<Sequence> record(
std::vector<std::shared_ptr<Tensor>> tensors,
TArgs&&... params)
std::vector<std::shared_ptr<Tensor>> tensors, TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(tensors, std::forward<TArgs>(params)...) };
return this->record(op);
}
template<typename T, typename... TArgs>
std::shared_ptr<Sequence> record(std::shared_ptr<Algorithm> algorithm,
TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(algorithm,
std::forward<TArgs>(params)...) };
return this->record(op);
}
@ -99,34 +82,15 @@ class Sequence : public std::enable_shared_from_this<Sequence>
std::shared_ptr<Sequence> eval(std::vector<std::shared_ptr<Tensor>> tensors,
TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(tensors, std::forward<TArgs>(params)...) };
// TODO: Aim to be able to handle errors when returning without throw
// except
return this->eval(op);
}
// Needded as otherise can't use initialiser list
template<typename T, typename... TArgs>
std::shared_ptr<Sequence> eval(std::shared_ptr<Algorithm> algorithm,
TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(algorithm,
std::forward<TArgs>(params)...) };
return this->eval(op);
}
@ -151,32 +115,15 @@ class Sequence : public std::enable_shared_from_this<Sequence>
std::vector<std::shared_ptr<Tensor>> tensors,
TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(tensors, std::forward<TArgs>(params)...) };
return this->evalAsync(op);
}
// Needed as otherwise it's not possible to use initializer lists
template<typename T, typename... TArgs>
std::shared_ptr<Sequence> evalAsync(std::shared_ptr<Algorithm> algorithm,
TArgs&&... params)
{
KP_LOG_DEBUG("Kompute Sequence record function started");
static_assert(std::is_base_of<OpBase, T>::value,
"Kompute Sequence record(...) template only valid with "
"OpBase derived classes");
KP_LOG_DEBUG("Kompute Sequence creating OpBase derived class instance");
std::shared_ptr<T> op{ new T(algorithm,
std::forward<TArgs>(params)...) };
return this->evalAsync(op);
}
@ -220,6 +167,8 @@ class Sequence : public std::enable_shared_from_this<Sequence>
bool isInit();
void rerecord();
/**
* Returns true if the sequence is currently running - mostly used for async
* workloads.

83
test/TestSequence.cpp
View file

@ -17,3 +17,86 @@ TEST(TestSequence, SequenceDestructorViaManager)
EXPECT_FALSE(sq->isInit());
}
TEST(TestSequence, SequenceDestructorOutsideManagerExplicit)
{
std::shared_ptr<kp::Sequence> sq = nullptr;
{
kp::Manager mgr;
sq = mgr.sequence();
EXPECT_TRUE(sq->isInit());
sq->destroy();
EXPECT_FALSE(sq->isInit());
}
EXPECT_FALSE(sq->isInit());
}
TEST(TestSequence, SequenceDestructorOutsideManagerImplicit)
{
kp::Manager mgr;
std::weak_ptr<kp::Sequence> sqWeak;
{
std::shared_ptr<kp::Sequence> sq = mgr.sequence();
sqWeak = sq;
EXPECT_TRUE(sq->isInit());
}
EXPECT_FALSE(sqWeak.lock());
}
TEST(TestSequence, RerecordSequence)
{
kp::Manager mgr;
std::shared_ptr<kp::Sequence> sq = mgr.sequence();
std::shared_ptr<kp::Tensor> tensorA = mgr.tensor({1, 2, 3});
std::shared_ptr<kp::Tensor> tensorB = mgr.tensor({2, 2, 2});
std::shared_ptr<kp::Tensor> tensorOut = mgr.tensor({0, 0, 0});
sq->eval<kp::OpTensorSyncDevice>({ tensorA, tensorB, tensorOut });
std::vector<uint32_t> spirv = kp::Shader::compile_source(R"(
#version 450
layout (local_size_x = 1) in;
// The input tensors bind index is relative to index in parameter passed
layout(set = 0, binding = 0) buffer bina { float tina[]; };
layout(set = 0, binding = 1) buffer binb { float tinb[]; };
layout(set = 0, binding = 2) buffer bout { float tout[]; };
void main() {
uint index = gl_GlobalInvocationID.x;
tout[index] = tina[index] * tinb[index];
}
)");
std::shared_ptr<kp::Algorithm> algo =
mgr.algorithm({tensorA, tensorB, tensorOut}, spirv);
sq->record<kp::OpAlgoDispatch>(algo)
->record<kp::OpTensorSyncLocal>({tensorA, tensorB, tensorOut});
sq->eval();
EXPECT_EQ(tensorOut->data(), std::vector<float>({2, 4, 6}));
algo->rebuild({tensorOut, tensorA, tensorB}, spirv);
// Refresh and trigger a rerecord
sq->rerecord();
sq->eval();
EXPECT_EQ(tensorB->data(), std::vector<float>({2, 8, 18}));
}