From 598c843545e5460f5f773550cf04de7ecc5c39cb Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Tue, 20 Jul 2021 21:53:50 +0100 Subject: [PATCH] Updated glslang as core dependency --- .gitignore | 2 +- .gitmodules | 4 - Makefile | 2 +- python/test/requirements-dev.txt | 1 + test/CMakeLists.txt | 35 +---- test/TestShaderResources.cpp | 68 --------- test/utils/kompute_test/Shader.cpp | 219 ++--------------------------- test/utils/kompute_test/Shader.hpp | 54 +------ 8 files changed, 23 insertions(+), 362 deletions(-) delete mode 100644 test/TestShaderResources.cpp diff --git a/.gitignore b/.gitignore index 4d67efdd8..f72d9ff7e 100644 --- a/.gitignore +++ b/.gitignore @@ -186,5 +186,5 @@ release/ # Kompute swiftshader/ vk_swiftshader_icd.json - +tmp_kp_shader.comp.spv diff --git a/.gitmodules b/.gitmodules index eeb2b007d..3c710d619 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,10 +14,6 @@ path = python/pybind11 url = https://github.com/pybind/pybind11 branch = v2.6.1 -[submodule "external/glslang"] - path = external/glslang - url = https://github.com/KhronosGroup/glslang/ - branch = 11.1.0 [submodule "external/fmt"] path = external/fmt url = https://github.com/fmtlib/fmt diff --git a/Makefile b/Makefile index f9498f8ab..c912f95c7 100644 --- a/Makefile +++ b/Makefile @@ -194,7 +194,7 @@ build_single_header: "single_include/kompute/Kompute.hpp" win_build_xxd: - cd external/bin/ && gcc -o xxd.exe xxd.c -DCYGWIN + cd external/bin/ && gcc.exe -o xxd.exe xxd.c -DCYGWIN format: $(CLANG_FORMAT_BIN) -i -style="{BasedOnStyle: mozilla, IndentWidth: 4}" src/*.cpp src/include/kompute/*.hpp test/*cpp diff --git a/python/test/requirements-dev.txt b/python/test/requirements-dev.txt index 8d5576a72..e44a9b538 100644 --- a/python/test/requirements-dev.txt +++ b/python/test/requirements-dev.txt @@ -1,3 +1,4 @@ pyshader==0.7.0 numpy==1.19.5 pytest==6.2.1 +pyshaderc==1.1.2 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8c4731a7e..ebe565b1a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -32,7 +32,7 @@ if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD) target_link_libraries(test_kompute gtest_main) else() - target_link_libraries(test_kompute PRIVATE + target_link_libraries(test_kompute GTest::gtest) endif() @@ -40,39 +40,6 @@ target_link_libraries(test_kompute kompute) add_test(NAME test_kompute COMMAND test_kompute) -##################################################### -#################### GLSLANG ####################### -##################################################### - -if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD) - add_subdirectory(${PROJECT_SOURCE_DIR}/external/glslang - ${CMAKE_CURRENT_BINARY_DIR}/kompute_glslang) - - target_include_directories( - test_kompute PRIVATE - ${PROJECT_SOURCE_DIR}/external/glslang) - - target_link_libraries(test_kompute - # Not including hlsl support - # HLSL - # glslang includes OGLCompiler, OSDependent, MachineIndependent - glslang - SPIRV) -else() - find_package(glslang CONFIG REQUIRED) - - target_include_directories( - test_kompute PRIVATE - ${GLSLANG_GENERATED_INCLUDEDIR}) - - target_link_libraries(test_kompute - # Not including hlsl support - # glslang::HLSL - # Adding explicit dependencies to match above - glslang::glslang - glslang::SPIRV) -endif() - ##################################################### #################### CODECOV ####################### diff --git a/test/TestShaderResources.cpp b/test/TestShaderResources.cpp deleted file mode 100644 index 41a979f6a..000000000 --- a/test/TestShaderResources.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include "gtest/gtest.h" - -#include "kompute/Kompute.hpp" - -#include "kompute_test/Shader.hpp" - -static const std::string shaderString = (R"( - #version 450 - - layout (local_size_x = 1) in; - - // The input tensors bind index is relative to index in parameter passed - layout(set = 0, binding = 0) buffer bina { float tina[]; }; - layout(set = 0, binding = 1) buffer binb { float tinb[]; }; - layout(set = 0, binding = 2) buffer bout { float tout[]; }; - - void main() { - uint index = gl_GlobalInvocationID.x; - - int i = 1; - - while(i < 200) { - tout[index] += tina[index] * tinb[index]; - i++; - } - } -)"); - -void -compileShaderWithGivenResources(const std::string shaderString, - const TBuiltInResource resources) -{ - kp_test_utils::Shader::compileSource( - shaderString, - std::string("main"), - std::vector>({}), - resources); -} - -TEST(TestShaderResources, TestNoMaxLight) -{ - TBuiltInResource noMaxLightResources = kp_test_utils::Shader::defaultResource; - noMaxLightResources.maxLights = 0; - - EXPECT_NO_THROW( - compileShaderWithGivenResources(shaderString, noMaxLightResources)); -} - -TEST(TestShaderResources, TestSmallComputeWorkGroupSizeX) -{ - TBuiltInResource smallComputeWorkGroupSizeXResources = - kp_test_utils::Shader::defaultResource; - smallComputeWorkGroupSizeXResources.maxComputeWorkGroupSizeX = 0; - - ASSERT_THROW(compileShaderWithGivenResources( - shaderString, smallComputeWorkGroupSizeXResources), - std::runtime_error); -} - -TEST(TestShaderResources, TestNoWhileLoopLimit) -{ - TBuiltInResource noWhileLoopLimitResources = kp_test_utils::Shader::defaultResource; - noWhileLoopLimitResources.limits.whileLoops = 0; - - ASSERT_THROW( - compileShaderWithGivenResources(shaderString, noWhileLoopLimitResources), - std::runtime_error); -} diff --git a/test/utils/kompute_test/Shader.cpp b/test/utils/kompute_test/Shader.cpp index 2fb9209a7..a4791432e 100644 --- a/test/utils/kompute_test/Shader.cpp +++ b/test/utils/kompute_test/Shader.cpp @@ -5,214 +5,23 @@ namespace kp_test_utils { -std::vector -Shader::compileSources( - const std::vector& sources, - const std::vector& files, - const std::string& entryPoint, - std::vector> definitions, - const TBuiltInResource& resources) -{ - - // Initialize glslang library. - glslang::InitializeProcess(); - - // Currently we don't support other shader types nor plan to - const EShLanguage language = EShLangCompute; - glslang::TShader shader(language); - - std::vector filesCStr(files.size()), - sourcesCStr(sources.size()); - for (size_t i = 0; i < sources.size(); i++) - sourcesCStr[i] = sources[i].c_str(); - - if (files.size() > 1) { - assert(files.size() == sources.size()); - for (size_t i = 0; i < files.size(); i++) - filesCStr[i] = files[i].c_str(); - shader.setStringsWithLengthsAndNames( - sourcesCStr.data(), nullptr, filesCStr.data(), filesCStr.size()); - } else { - filesCStr = { "" }; - shader.setStringsWithLengthsAndNames( - sourcesCStr.data(), nullptr, filesCStr.data(), sourcesCStr.size()); - } - - shader.setEntryPoint(entryPoint.c_str()); - shader.setSourceEntryPoint(entryPoint.c_str()); - - std::string info_log = ""; - const EShMessages messages = static_cast( - EShMsgDefault | EShMsgVulkanRules | EShMsgSpvRules); - if (!shader.parse(&resources, 100, false, messages)) { - info_log = std::string(shader.getInfoLog()) + "\n" + - std::string(shader.getInfoDebugLog()); - KP_LOG_ERROR("Kompute Shader Error: {}", info_log); - throw std::runtime_error(info_log); - } - - // Add shader to new program object. - glslang::TProgram program; - program.addShader(&shader); - // Link program. - if (!program.link(messages)) { - info_log = std::string(program.getInfoLog()) + "\n" + - std::string(program.getInfoDebugLog()); - KP_LOG_ERROR("Kompute Shader Error: {}", info_log); - throw std::runtime_error(info_log); - } - - // Save any info log that was generated. - if (shader.getInfoLog()) { - info_log += std::string(shader.getInfoLog()) + "\n" + - std::string(shader.getInfoDebugLog()) + "\n"; - KP_LOG_INFO("Kompute Shader Information: {}", info_log); - } - - glslang::TIntermediate* intermediate = program.getIntermediate(language); - // Translate to SPIRV. - if (!intermediate) { - info_log += "Failed to get shared intermediate code.\n"; - KP_LOG_ERROR("Kompute Shader Error: {}", info_log); - throw std::runtime_error(info_log); - } - - spv::SpvBuildLogger logger; - std::vector spirv; - glslang::GlslangToSpv(*intermediate, spirv, &logger); - - if (shader.getInfoLog()) { - info_log += logger.getAllMessages() + "\n"; - KP_LOG_DEBUG("Kompute Shader all result messages: {}", info_log); - } - - // Shutdown glslang library. - glslang::FinalizeProcess(); - - return spirv; -} - std::vector Shader::compileSource( - const std::string& source, - const std::string& entryPoint, - std::vector> definitions, - const TBuiltInResource& resource) + const std::string& source) { - return compileSources({ source }, - std::vector({}), - entryPoint, - definitions, - resource); + std::string cmd = "glslc -fshader-stage=compute -o tmp_kp_shader.comp.spv - << END\n" + source + "\nEND"; + system(cmd.c_str()); + + std::ifstream fileStream("tmp_kp_shader.comp.spv", + std::ios::binary | std::ios::in | std::ios::ate); + + size_t shaderFileSize = fileStream.tellg(); + fileStream.seekg(0, std::ios::beg); + char* shaderDataRaw = new char[shaderFileSize]; + fileStream.read(shaderDataRaw, shaderFileSize); + fileStream.close(); + std::vector tmpResult = {(uint32_t*)shaderDataRaw, (uint32_t*)(shaderDataRaw + shaderFileSize)}; + return tmpResult; } -const TBuiltInResource Shader::defaultResource = { - /* .MaxLights = */ 0, - /* .MaxClipPlanes = */ 0, - /* .MaxTextureUnits = */ 0, - /* .MaxTextureCoords = */ 0, - /* .MaxVertexAttribs = */ 64, - /* .MaxVertexUniformComponents = */ 4096, - /* .MaxVaryingFloats = */ 64, - /* .MaxVertexTextureImageUnits = */ 0, - /* .MaxCombinedTextureImageUnits = */ 0, - /* .MaxTextureImageUnits = */ 0, - /* .MaxFragmentUniformComponents = */ 0, - /* .MaxDrawBuffers = */ 0, - /* .MaxVertexUniformVectors = */ 128, - /* .MaxVaryingVectors = */ 8, - /* .MaxFragmentUniformVectors = */ 0, - /* .MaxVertexOutputVectors = */ 16, - /* .MaxFragmentInputVectors = */ 0, - /* .MinProgramTexelOffset = */ -8, - /* .MaxProgramTexelOffset = */ 7, - /* .MaxClipDistances = */ 8, - /* .MaxComputeWorkGroupCountX = */ 65535, - /* .MaxComputeWorkGroupCountY = */ 65535, - /* .MaxComputeWorkGroupCountZ = */ 65535, - /* .MaxComputeWorkGroupSizeX = */ 1024, - /* .MaxComputeWorkGroupSizeY = */ 1024, - /* .MaxComputeWorkGroupSizeZ = */ 64, - /* .MaxComputeUniformComponents = */ 1024, - /* .MaxComputeTextureImageUnits = */ 16, - /* .MaxComputeImageUniforms = */ 8, - /* .MaxComputeAtomicCounters = */ 8, - /* .MaxComputeAtomicCounterBuffers = */ 1, - /* .MaxVaryingComponents = */ 60, - /* .MaxVertexOutputComponents = */ 64, - /* .MaxGeometryInputComponents = */ 64, - /* .MaxGeometryOutputComponents = */ 128, - /* .MaxFragmentInputComponents = */ 0, - /* .MaxImageUnits = */ 0, - /* .MaxCombinedImageUnitsAndFragmentOutputs = */ 0, - /* .MaxCombinedShaderOutputResources = */ 8, - /* .MaxImageSamples = */ 0, - /* .MaxVertexImageUniforms = */ 0, - /* .MaxTessControlImageUniforms = */ 0, - /* .MaxTessEvaluationImageUniforms = */ 0, - /* .MaxGeometryImageUniforms = */ 0, - /* .MaxFragmentImageUniforms = */ 0, - /* .MaxCombinedImageUniforms = */ 0, - /* .MaxGeometryTextureImageUnits = */ 0, - /* .MaxGeometryOutputVertices = */ 256, - /* .MaxGeometryTotalOutputComponents = */ 1024, - /* .MaxGeometryUniformComponents = */ 1024, - /* .MaxGeometryVaryingComponents = */ 64, - /* .MaxTessControlInputComponents = */ 128, - /* .MaxTessControlOutputComponents = */ 128, - /* .MaxTessControlTextureImageUnits = */ 0, - /* .MaxTessControlUniformComponents = */ 1024, - /* .MaxTessControlTotalOutputComponents = */ 4096, - /* .MaxTessEvaluationInputComponents = */ 128, - /* .MaxTessEvaluationOutputComponents = */ 128, - /* .MaxTessEvaluationTextureImageUnits = */ 16, - /* .MaxTessEvaluationUniformComponents = */ 1024, - /* .MaxTessPatchComponents = */ 120, - /* .MaxPatchVertices = */ 32, - /* .MaxTessGenLevel = */ 64, - /* .MaxViewports = */ 16, - /* .MaxVertexAtomicCounters = */ 0, - /* .MaxTessControlAtomicCounters = */ 0, - /* .MaxTessEvaluationAtomicCounters = */ 0, - /* .MaxGeometryAtomicCounters = */ 0, - /* .MaxFragmentAtomicCounters = */ 0, - /* .MaxCombinedAtomicCounters = */ 8, - /* .MaxAtomicCounterBindings = */ 1, - /* .MaxVertexAtomicCounterBuffers = */ 0, - /* .MaxTessControlAtomicCounterBuffers = */ 0, - /* .MaxTessEvaluationAtomicCounterBuffers = */ 0, - /* .MaxGeometryAtomicCounterBuffers = */ 0, - /* .MaxFragmentAtomicCounterBuffers = */ 0, - /* .MaxCombinedAtomicCounterBuffers = */ 1, - /* .MaxAtomicCounterBufferSize = */ 16384, - /* .MaxTransformFeedbackBuffers = */ 4, - /* .MaxTransformFeedbackInterleavedComponents = */ 64, - /* .MaxCullDistances = */ 8, - /* .MaxCombinedClipAndCullDistances = */ 8, - /* .MaxSamples = */ 4, - /* .maxMeshOutputVerticesNV = */ 256, - /* .maxMeshOutputPrimitivesNV = */ 512, - /* .maxMeshWorkGroupSizeX_NV = */ 32, - /* .maxMeshWorkGroupSizeY_NV = */ 1, - /* .maxMeshWorkGroupSizeZ_NV = */ 1, - /* .maxTaskWorkGroupSizeX_NV = */ 32, - /* .maxTaskWorkGroupSizeY_NV = */ 1, - /* .maxTaskWorkGroupSizeZ_NV = */ 1, - /* .maxMeshViewCountNV = */ 4, - /* .maxDualSourceDrawBuffersEXT = */ 1, - - /* .limits = */ - { - /* .nonInductiveForLoops = */ 1, - /* .whileLoops = */ 1, - /* .doWhileLoops = */ 1, - /* .generalUniformIndexing = */ 1, - /* .generalAttributeMatrixVectorIndexing = */ 1, - /* .generalVaryingIndexing = */ 1, - /* .generalSamplerIndexing = */ 1, - /* .generalVariableIndexing = */ 1, - /* .generalConstantMatrixVectorIndexing = */ 1, - } -}; - } diff --git a/test/utils/kompute_test/Shader.hpp b/test/utils/kompute_test/Shader.hpp index a24234e30..b6df40dce 100644 --- a/test/utils/kompute_test/Shader.hpp +++ b/test/utils/kompute_test/Shader.hpp @@ -3,15 +3,6 @@ #include #include -#ifdef USE_EXTERNAL_GLSLANG -#include -#else -#include -#endif - -#include -#include - namespace kp_test_utils { /** @@ -20,52 +11,17 @@ namespace kp_test_utils { class Shader { public: - // The default resource limit for the GLSL compiler, can be overwritten - // Has been adopted by: - // https://github.com/KhronosGroup/glslang/blob/master/StandAlone/ResourceLimits.cpp - const static TBuiltInResource defaultResource; - /** - * Compile multiple sources with optional filenames. Currently this function - * uses the glslang C++ interface which is not thread safe so this funciton - * should not be called from multiple threads concurrently. If you have a - * online shader processing multithreading use-case that can't use offline - * compilation please open an issue. - * - * @param sources A list of raw glsl shaders in string format - * @param files A list of file names respective to each of the sources - * @param entryPoint The function name to use as entry point - * @param definitions List of pairs containing key value definitions - * @param resourcesLimit A list that contains the resource limits for the - * GLSL compiler - * @return The compiled SPIR-V binary in unsigned int32 format - */ - static std::vector compileSources( - const std::vector& sources, - const std::vector& files = {}, - const std::string& entryPoint = "main", - std::vector> definitions = {}, - const TBuiltInResource& resources = Shader::defaultResource); - - /** - * Compile a single glslang source from string value. Currently this - * function uses the glslang C++ interface which is not thread safe so this - * funciton should not be called from multiple threads concurrently. If you - * have a online shader processing multithreading use-case that can't use - * offline compilation please open an issue. + * Compile a single glslang source from string value. This is only meant + * to be used for testing as it's non threadsafe, and it had to be removed + * from the glslang dependency and now can only run the CLI directly due to + * license issues: see https://github.com/EthicalML/vulkan-kompute/pull/235 * * @param source An individual raw glsl shader in string format - * @param entryPoint The function name to use as entry point - * @param definitions List of pairs containing key value definitions - * @param resourcesLimit A list that contains the resource limits for the - * GLSL compiler * @return The compiled SPIR-V binary in unsigned int32 format */ static std::vector compileSource( - const std::string& source, - const std::string& entryPoint = "main", - std::vector> definitions = {}, - const TBuiltInResource& resources = Shader::defaultResource); + const std::string& source); }; }