diff --git a/CMakeLists.txt b/CMakeLists.txt index c31fe0b0a..76aa13073 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,7 @@ option(KOMPUTE_OPT_INSTALL "Enable if you want to enable installation" 0) # Build options option(KOMPUTE_OPT_BUILD_PYTHON "Enable if you want to build python bindings" 0) option(KOMPUTE_OPT_ENABLE_SPDLOG "Extra compile flags for Kompute, see docs for full list" 0) +option(KOMPUTE_OPT_DISABLE_GLSLANG "Skip compilation of glslang utilities" 0) option(KOMPUTE_OPT_REPO_SUBMODULE_BUILD, "Use the submodule repos instead of external package manager" 0) option(KOMPUTE_OPT_ANDOID_BUILD "Enable android compilation flags required" 0) option(KOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS "Explicitly disable debug layers even on debug" 0) @@ -24,7 +25,10 @@ set(KOMPUTE_EXTRA_CXX_FLAGS "" CACHE STRING "Extra compile flags for Kompute, se if(KOMPUTE_OPT_ENABLE_SPDLOG) set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_ENABLE_SPDLOG=1") - set(SPDLOG_INSTALL, 1) + if(KOMPUTE_OPT_INSTALL) + # Enable install parameters for spdlog (overrides parameters passed) + set(SPDLOG_INSTALL ON CACHE BOOL "Enables install of glslang" FORCE) + endif() endif() if(KOMPUTE_OPT_ANDOID_BUILD) @@ -39,6 +43,16 @@ if(KOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS) set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_DISABLE_VK_DEBUG_LAYERS=1") endif() +if(KOMPUTE_OPT_DISABLE_GLSLANG) + set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_DISABLE_GLSLANG=1") +else() + if(KOMPUTE_OPT_INSTALL) + # Enable install parameters for glslang (overrides parameters passed) + set(ENABLE_GLSLANG_INSTALL ON CACHE BOOL "Enables install of glslang" FORCE) + set(BUILD_SHARED_LIBS ON CACHE BOOL "Enables build of shared libraries" FORCE) + endif() +endif() + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG=1 ${KOMPUTE_EXTRA_CXX_FLAGS} -DUSE_DEBUG_EXTENTIONS") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DRELEASE=1 ${KOMPUTE_EXTRA_CXX_FLAGS}") diff --git a/Makefile b/Makefile index ccda4e2c3..37abad73b 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,6 @@ mk_cmake: -DKOMPUTE_OPT_ENABLE_SPDLOG=1 \ -DSPDLOG_INSTALL=1 \ -DKOMPUTE_OPT_CODE_COVERAGE=1 \ - -DSHADERC_SKIP_TESTS=1 \ -G "Unix Makefiles" mk_build_all: diff --git a/single_include/AggregateHeaders.cpp b/single_include/AggregateHeaders.cpp index 57ab728cc..ee5f89bf7 100644 --- a/single_include/AggregateHeaders.cpp +++ b/single_include/AggregateHeaders.cpp @@ -1,5 +1,6 @@ #pragma once #include "kompute/Core.hpp" +#include "kompute/ShaderUtil.hpp" #include "kompute/shaders/shaderopmult.hpp" #include "kompute/shaders/shaderlogisticregression.hpp" #include "kompute/Manager.hpp" diff --git a/single_include/kompute/Kompute.hpp b/single_include/kompute/Kompute.hpp index bde529400..8471639e6 100755 --- a/single_include/kompute/Kompute.hpp +++ b/single_include/kompute/Kompute.hpp @@ -107,6 +107,101 @@ extern py::object kp_debug, kp_info, kp_warning, kp_error; #endif // KOMPUTE_SPDLOG_ENABLED #endif // KOMPUTE_LOG_OVERRIDE +#include +#include + +#include +#include +#include + +namespace kp { + +static std::vector spirv_from_sources(const std::vector& sources, + const std::vector& files = {}, + const std::string& entryPoint = "main", + std::vector> definitions = {}) { + + // Initialize glslang library. + glslang::InitializeProcess(); + + // Currently we don't support other shader types nor plan to + const EShLanguage language = EShLangCompute; + glslang::TShader shader(language); + + std::vector filesCStr(files.size()), sourcesCStr(sources.size()); + for (size_t i = 0; i < sources.size(); i++) sourcesCStr[i] = sources[i].c_str(); + + if (files.size() > 1) { + assert(files.size() == sources.size()); + for (size_t i = 0; i < files.size(); i++) filesCStr[i] = files[i].c_str(); + shader.setStringsWithLengthsAndNames(sourcesCStr.data(), nullptr, filesCStr.data(), filesCStr.size()); + } + else { + filesCStr = {""}; + shader.setStringsWithLengthsAndNames(sourcesCStr.data(), nullptr, filesCStr.data(), sourcesCStr.size()); + } + + shader.setEntryPoint(entryPoint.c_str()); + shader.setSourceEntryPoint(entryPoint.c_str()); + + std::string info_log = ""; + const EShMessages messages = static_cast(EShMsgDefault | EShMsgVulkanRules | EShMsgSpvRules); + if (!shader.parse(&glslang::DefaultTBuiltInResource, 100, false, messages)) + { + info_log = std::string(shader.getInfoLog()) + "\n" + std::string(shader.getInfoDebugLog()); + SPDLOG_ERROR(info_log); + throw std::runtime_error(info_log); + } + + // Add shader to new program object. + glslang::TProgram program; + program.addShader(&shader); + // Link program. + if (!program.link(messages)) + { + info_log = std::string(program.getInfoLog()) + "\n" + std::string(program.getInfoDebugLog()); + SPDLOG_ERROR(info_log); + throw std::runtime_error(info_log); + } + + // Save any info log that was generated. + if (shader.getInfoLog()) + { + info_log += std::string(shader.getInfoLog()) + "\n" + std::string(shader.getInfoDebugLog()) + "\n"; + SPDLOG_INFO(info_log); + } + + glslang::TIntermediate *intermediate = program.getIntermediate(language); + // Translate to SPIRV. + if (!intermediate) + { + info_log += "Failed to get shared intermediate code.\n"; + SPDLOG_ERROR(info_log); + throw std::runtime_error(info_log); + } + + spv::SpvBuildLogger logger; + std::vector spirv; + glslang::GlslangToSpv(*intermediate, spirv, &logger); + + if (shader.getInfoLog()) + { + info_log += logger.getAllMessages() + "\n"; + SPDLOG_DEBUG(info_log); + } + + // Shutdown glslang library. + glslang::FinalizeProcess(); + + return std::vector((char*)spirv.data(), (char*)(spirv.data()+spirv.size()) ); +} + +static std::vector spirv_from_source(const std::string& source) { + return spirv_from_sources({source}); +} + +} + /* THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5d9cbd6c3..567e18778 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -105,6 +105,45 @@ if(KOMPUTE_OPT_BUILD_SINGLE_HEADER) build_single_header) endif() +##################################################### +#################### GLSLANG ####################### +##################################################### + +if(NOT KOMPUTE_OPT_DISABLE_GLSLANG) + + if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD) + add_subdirectory(${PROJECT_SOURCE_DIR}/external/glslang + ${CMAKE_CURRENT_BINARY_DIR}/kompute_glslang) + + target_include_directories( + kompute PRIVATE + ${PROJECT_SOURCE_DIR}/external/glslang) + + target_link_libraries(kompute + # Not including hlsl support + # HLSL + # glslang includes OGLCompiler, OSDependent, MachineIndependent + glslang + SPIRV + glslang-default-resource-limits) + else() + find_package(glslang CONFIG REQUIRED) + + target_include_directories( + kompute PRIVATE + ${GLSLANG_GENERATED_INCLUDEDIR}) + + target_link_libraries(kompute + # Not including hlsl support + # glslang::HLSL + # Adding explicit dependencies to match above + glslang + SPIRV + glslang-default-resource-limits) + endif() +endif() + + add_library(kompute::kompute ALIAS kompute) if(KOMPUTE_OPT_INSTALL) diff --git a/test/TestUtils.cpp b/src/include/kompute/ShaderUtil.hpp similarity index 90% rename from test/TestUtils.cpp rename to src/include/kompute/ShaderUtil.hpp index f0931b276..a6b994a6f 100644 --- a/test/TestUtils.cpp +++ b/src/include/kompute/ShaderUtil.hpp @@ -1,13 +1,16 @@ +#pragma once #include #include -#include "kompute/Kompute.hpp" +#include "Core.hpp" #include #include #include +namespace kp { + static std::vector spirv_from_sources(const std::vector& sources, const std::vector& files = {}, const std::string& entryPoint = "main", @@ -41,10 +44,10 @@ static std::vector spirv_from_sources(const std::vector& sour if (!shader.parse(&glslang::DefaultTBuiltInResource, 100, false, messages)) { info_log = std::string(shader.getInfoLog()) + "\n" + std::string(shader.getInfoDebugLog()); + SPDLOG_ERROR(info_log); throw std::runtime_error(info_log); } - // Add shader to new program object. glslang::TProgram program; program.addShader(&shader); @@ -60,6 +63,7 @@ static std::vector spirv_from_sources(const std::vector& sour if (shader.getInfoLog()) { info_log += std::string(shader.getInfoLog()) + "\n" + std::string(shader.getInfoDebugLog()) + "\n"; + SPDLOG_INFO(info_log); } glslang::TIntermediate *intermediate = program.getIntermediate(language); @@ -74,7 +78,12 @@ static std::vector spirv_from_sources(const std::vector& sour spv::SpvBuildLogger logger; std::vector spirv; glslang::GlslangToSpv(*intermediate, spirv, &logger); - info_log += logger.getAllMessages() + "\n"; + + if (shader.getInfoLog()) + { + info_log += logger.getAllMessages() + "\n"; + SPDLOG_DEBUG(info_log); + } // Shutdown glslang library. glslang::FinalizeProcess(); @@ -82,7 +91,8 @@ static std::vector spirv_from_sources(const std::vector& sour return std::vector((char*)spirv.data(), (char*)(spirv.data()+spirv.size()) ); } -static std::vector spirv_from_string(const std::string& source) { +static std::vector spirv_from_source(const std::string& source) { return spirv_from_sources({source}); } +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d326f5317..a30792aad 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -38,45 +38,6 @@ target_link_libraries(test_kompute PRIVATE kompute) add_test(NAME test_kompute COMMAND test_kompute) -##################################################### -#################### PYSHADERC ####################### -##################################################### - -set(SHADERC_SKIP_TESTS 1) - -if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD) - add_subdirectory(${PROJECT_SOURCE_DIR}/external/glslang - ${CMAKE_CURRENT_BINARY_DIR}/kompute_glslang) - - target_include_directories( - test_kompute PRIVATE - ${PROJECT_SOURCE_DIR}/external/glslang) - - target_link_libraries(test_kompute PRIVATE - # Not including hlsl support - # HLSL - # glslang includes OGLCompiler, OSDependent, MachineIndependent - glslang - SPIRV - glslang-default-resource-limits - MachineIndependent) -else() - find_package(glslang CONFIG REQUIRED) - - target_include_directories( - test_kompute PRIVATE - ${GLSLANG_GENERATED_INCLUDEDIR}) - - target_link_libraries(test_kompute PRIVATE - # Not including hlsl support - # glslang::HLSL - # Adding explicit dependencies to match above - glslang - SPIRV - glslang-default-resource-limits - MachineIndependent) -endif() - ##################################################### #################### CODECOV ####################### ##################################################### diff --git a/test/TestAsyncOperations.cpp b/test/TestAsyncOperations.cpp index 9e1cdfd88..ab3254443 100644 --- a/test/TestAsyncOperations.cpp +++ b/test/TestAsyncOperations.cpp @@ -5,8 +5,6 @@ #include "kompute/Kompute.hpp" -#include "TestUtils.cpp" - TEST(TestAsyncOperations, TestManagerParallelExecution) { // This test is built for NVIDIA 1650. It assumes: @@ -57,7 +55,7 @@ TEST(TestAsyncOperations, TestManagerParallelExecution) for (uint32_t i = 0; i < numParallel; i++) { mgr.evalOpDefault( - { inputsSyncB[i] }, spirv_from_string(shader)); + { inputsSyncB[i] }, kp::spirv_from_source(shader)); } auto endSync = std::chrono::high_resolution_clock::now(); @@ -91,7 +89,7 @@ TEST(TestAsyncOperations, TestManagerParallelExecution) mgrAsync.evalOpAsync( { inputsAsyncB[i] }, "async" + std::to_string(i), - spirv_from_string(shader)); + kp::spirv_from_source(shader)); } for (uint32_t i = 0; i < numParallel; i++) { @@ -154,10 +152,10 @@ TEST(TestAsyncOperations, TestManagerAsyncExecution) mgr.rebuild({ tensorA, tensorB }); mgr.evalOpAsync( - { tensorA }, "asyncOne", spirv_from_string(shader)); + { tensorA }, "asyncOne", kp::spirv_from_source(shader)); mgr.evalOpAsync( - { tensorB }, "asyncTwo", spirv_from_string(shader)); + { tensorB }, "asyncTwo", kp::spirv_from_source(shader)); mgr.evalOpAwait("asyncOne"); mgr.evalOpAwait("asyncTwo"); diff --git a/test/TestDestroy.cpp b/test/TestDestroy.cpp index 98ded696b..6e5ba3376 100644 --- a/test/TestDestroy.cpp +++ b/test/TestDestroy.cpp @@ -3,8 +3,6 @@ #include "kompute/Kompute.hpp" -#include "TestUtils.cpp" - TEST(TestDestroy, TestDestroyTensorSingle) { std::shared_ptr tensorA{ new kp::Tensor({ 0, 0, 0 }) }; @@ -30,7 +28,7 @@ TEST(TestDestroy, TestDestroyTensorSingle) sq->begin(); sq->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq->end(); sq->eval(); @@ -73,7 +71,7 @@ TEST(TestDestroy, TestDestroyTensorVector) sq->begin(); sq->record( - { tensorA, tensorB }, spirv_from_string(shader)); + { tensorA, tensorB }, kp::spirv_from_source(shader)); sq->end(); sq->eval(); @@ -138,7 +136,7 @@ TEST(TestDestroy, TestDestroySequenceSingle) sq->begin(); sq->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq->end(); sq->eval(); @@ -178,14 +176,14 @@ TEST(TestDestroy, TestDestroySequenceVector) sq1 = mgr.sequence("One"); sq1->begin(); sq1->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq1->end(); sq1->eval(); sq2 = mgr.sequence("Two"); sq2->begin(); sq2->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq2->end(); sq2->eval(); @@ -220,11 +218,11 @@ TEST(TestDestroy, TestDestroySequenceNameSingleInsideManager) mgr.evalOp( { tensorA }, "one", - spirv_from_string(shader)); + kp::spirv_from_source(shader)); mgr.evalOp( { tensorA }, "two", - spirv_from_string(shader)); + kp::spirv_from_source(shader)); mgr.evalOpDefault({ tensorA }); @@ -259,7 +257,7 @@ TEST(TestDestroy, TestDestroySequenceNameSingleOutsideManager) sq1 = mgr.sequence("One"); sq1->begin(); sq1->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq1->end(); sq1->eval(); @@ -293,11 +291,11 @@ TEST(TestDestroy, TestDestroySequenceNameVectorInsideManager) mgr.evalOp( { tensorA }, "one", - spirv_from_string(shader)); + kp::spirv_from_source(shader)); mgr.evalOp( { tensorA }, "two", - spirv_from_string(shader)); + kp::spirv_from_source(shader)); mgr.evalOpDefault({ tensorA }); @@ -327,11 +325,11 @@ TEST(TestDestroy, TestDestroySequenceNameVectorOutsideManager) mgr.evalOp( { tensorA }, "one", - spirv_from_string(shader)); + kp::spirv_from_source(shader)); mgr.evalOp( { tensorA }, "two", - spirv_from_string(shader)); + kp::spirv_from_source(shader)); mgr.evalOpDefault({ tensorA }); @@ -361,7 +359,7 @@ TEST(TestDestroy, TestDestroySequenceNameDefaultOutsideManager) mgr.evalOpDefault( { tensorA }, - spirv_from_string(shader)); + kp::spirv_from_source(shader)); mgr.evalOpDefault({ tensorA }); diff --git a/test/TestMultipleAlgoExecutions.cpp b/test/TestMultipleAlgoExecutions.cpp index 81d4b2cd6..727e2d5b4 100644 --- a/test/TestMultipleAlgoExecutions.cpp +++ b/test/TestMultipleAlgoExecutions.cpp @@ -3,8 +3,6 @@ #include "kompute/Kompute.hpp" -#include "TestUtils.cpp" - TEST(TestMultipleAlgoExecutions, SingleSequenceRecord) { @@ -30,11 +28,11 @@ TEST(TestMultipleAlgoExecutions, SingleSequenceRecord) sq->begin(); sq->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq->record({ tensorA }); @@ -75,19 +73,19 @@ TEST(TestMultipleAlgoExecutions, MultipleCmdBufRecords) // Then perform the computations sq->begin(); sq->record({ tensorA }, - spirv_from_string(shader)); + kp::spirv_from_source(shader)); sq->end(); sq->eval(); sq->begin(); sq->record({ tensorA }, - spirv_from_string(shader)); + kp::spirv_from_source(shader)); sq->end(); sq->eval(); sq->begin(); sq->record({ tensorA }, - spirv_from_string(shader)); + kp::spirv_from_source(shader)); sq->end(); sq->eval(); @@ -124,7 +122,7 @@ TEST(TestMultipleAlgoExecutions, MultipleSequences) sq->begin(); sq->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq->end(); sq->eval(); @@ -137,7 +135,7 @@ TEST(TestMultipleAlgoExecutions, MultipleSequences) sq->begin(); sq->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq->end(); sq->eval(); @@ -150,7 +148,7 @@ TEST(TestMultipleAlgoExecutions, MultipleSequences) sq->begin(); sq->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq->end(); sq->eval(); @@ -208,7 +206,7 @@ TEST(TestMultipleAlgoExecutions, SingleRecordMultipleEval) sq->begin(); sq->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq->end(); @@ -267,7 +265,7 @@ TEST(TestMultipleAlgoExecutions, ManagerEvalMultSourceStrOpCreate) mgr.evalOpDefault( { tensorInA, tensorInB, tensorOut }, - spirv_from_string(shader)); + kp::spirv_from_source(shader)); mgr.evalOpDefault({ tensorOut }); @@ -310,7 +308,7 @@ TEST(TestMultipleAlgoExecutions, ManagerEvalMultSourceStrMgrCreate) mgr.evalOpDefault( { tensorInA, tensorInB, tensorOut }, - spirv_from_string(shader)); + kp::spirv_from_source(shader)); mgr.evalOpDefault({ tensorOut }); @@ -342,7 +340,7 @@ TEST(TestMultipleAlgoExecutions, SequenceAlgoDestroyOutsideManagerScope) sq->begin(); sq->record( - { tensorA }, spirv_from_string(shader)); + { tensorA }, kp::spirv_from_source(shader)); sq->end(); sq->eval(); diff --git a/test/TestOpAlgoLoopsPassingData.cpp b/test/TestOpAlgoLoopsPassingData.cpp index a410e4acf..70412ee7c 100644 --- a/test/TestOpAlgoLoopsPassingData.cpp +++ b/test/TestOpAlgoLoopsPassingData.cpp @@ -3,8 +3,6 @@ #include "kompute/Kompute.hpp" -#include "TestUtils.cpp" - TEST(TestProcessingIterations, IterateThroughMultipleSumAndCopies) { kp::Manager mgr; @@ -55,7 +53,7 @@ TEST(TestProcessingIterations, IterateThroughMultipleSumAndCopies) sq->record( { tensorA, tensorB }, - spirv_from_string(shader)); + kp::spirv_from_source(shader)); sq->record({ tensorB, tensorA }); sq->end(); diff --git a/test/TestOpShadersFromStringAndFile.cpp b/test/TestOpShadersFromStringAndFile.cpp index 73271fc6d..f64cc16ed 100644 --- a/test/TestOpShadersFromStringAndFile.cpp +++ b/test/TestOpShadersFromStringAndFile.cpp @@ -5,8 +5,6 @@ #include "kompute_test/shaders/shadertest_op_custom_shader.hpp" -#include "TestUtils.cpp" - TEST(TestOpAlgoBase, ShaderRawDataFromConstructor) { kp::Manager mgr; @@ -31,7 +29,7 @@ TEST(TestOpAlgoBase, ShaderRawDataFromConstructor) )"); mgr.evalOpDefault( - { tensorA, tensorB }, spirv_from_string(shader)); + { tensorA, tensorB }, kp::spirv_from_source(shader)); mgr.evalOpDefault({ tensorA, tensorB }); diff --git a/test/TestSpecializationConstant.cpp b/test/TestSpecializationConstant.cpp index 74274a029..a0a164901 100644 --- a/test/TestSpecializationConstant.cpp +++ b/test/TestSpecializationConstant.cpp @@ -1,10 +1,7 @@ - #include "gtest/gtest.h" #include "kompute/Kompute.hpp" -#include "TestUtils.cpp" - TEST(TestSpecializationConstants, TestTwoConstants) { std::shared_ptr tensorA{ new kp::Tensor({ 0, 0, 0 }) }; @@ -38,7 +35,7 @@ TEST(TestSpecializationConstants, TestTwoConstants) sq->begin(); sq->record( { tensorA, tensorB }, - spirv_from_string(shader), + kp::spirv_from_source(shader), kp::Workgroup(), spec); sq->end();